RucioKubernetesAdminGuide

Rucio Client-Server Pods

1) Create a directory for the Kubernetes files.

mkdir -p ~/Code/Kubernetes/Rucio/config && cd ~/Code/Kubernetes/Rucio/

2) Create the deployment.yaml.

`vi ~/Code/Kubernetes/Rucio/deployment.yaml` and paste the contents of this YAML:

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: rucio-server
  labels:
    app: rucio-server
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: rucio-server
    spec:
      volumes:
      - name: rucio-test-x509 
        secret:
          secretName: rucio-test-x509
          defaultMode: 256
      - name: rucio-test-etc
        configMap:
          name: rucio-test-etc
      - name: rucio-entrypoint
        configMap:
          name: rucio-entrypoint
      - name: rucio-httpd-conf
        configMap:
          name: rucio-httpd-conf
      - name: rucio-test-httpd
        configMap:
          name: rucio-test-httpd
      - name: rucio-host-pems
        secret:
          secretName: rucio-host-pems
          defaultMode: 256
      - name: postgresql-persistent-storage
        persistentVolumeClaim:
          claimName: postgresql-persistent-claim
      - name: rucio-ca-crt
        configMap:
          name: rucio-ca-crt
      containers:
        - name: postgres-container
          image: postgres:11.2
          env:
            - name: POSTGRES_USER
              value: rucio
            - name: POSTGRES_PASSWORD
              value: rucio
            - name: POSTGRES_DB
              value: rucio
            - name: PGDATA
              value: /var/lib/postgresql/data/pgdata
          ports:
            - containerPort: 5432
          volumeMounts:
            - name: postgresql-persistent-storage
              mountPath: /var/lib/postgresql/data
        - name: rucio-server
          image: rucio/rucio-server:release-1.19.4
          imagePullPolicy: Always
          ports:
            - containerPort: 443 
          env:
            - name: RUCIO_ENABLE_SSL
              value: "True"
          command: ["/bin/sh"]
          args: ["/tmp/entrypoint/installer.sh"]
          #livenessProbe:
          #  httpGet:
          #    path: /ping
          #    port: 80
          #  initialDelaySeconds: 10
          #  periodSeconds: 10
          #  timeoutSeconds: 3
          volumeMounts:
          - name: rucio-test-x509
            mountPath: /opt/rucio/x509
          - name: rucio-test-etc
            mountPath: /opt/rucio/etc
          - name: rucio-test-httpd
            mountPath: /tmp/httpd/conf.d
          - name: rucio-httpd-conf
            mountPath: /tmp/httpdconf
          - name: rucio-ca-crt
            mountPath: /opt/rucio/web
          - name: rucio-entrypoint
            mountPath: /tmp/entrypoint/
          - name: rucio-host-pems
            mountPath: /tmp/rucio-host-pems/
---
apiVersion: v1
kind: Service
metadata:
  name: rucio-server
  labels:
    app: rucio-server
  annotations:
    external-dns.alpha.kubernetes.io/hostname: rucio-k8s-test.grid.uchicago.edu
spec:
  type: LoadBalancer 
  ports:
    - port: 443
      protocol: TCP
      targetPort: 443
  selector:
    app: rucio-server
---
kind: PersistentVolume
apiVersion: v1
metadata:
  name: postgresql-persistent-storage 
  labels:
    type: local
spec:
  storageClassName: nfs-partitioner
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteMany
  hostPath:
    path: "/mnt/data"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: postgresql-persistent-claim 
spec:
  storageClassName: nfs-partitioner 
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 5Gi

3) Create the entrypoint shell script.

`vi ~/Code/Kubernetes/Rucio/installer.sh` and paste the contents of this shell script:

#!/bin/bash -e

if [ -f /opt/rucio/etc/rucio.cfg ]; then
    echo "rucio.cfg already mounted."
else
    echo "rucio.cfg not found. will generate one."
    j2 /tmp/rucio.cfg.j2 | sed '/^\s*$/d' > /opt/rucio/etc/rucio.cfg
fi

echo "=================== /opt/rucio/etc/rucio.cfg ============================"
cat /opt/rucio/etc/rucio.cfg
echo ""

#j2 /tmp/rucio.conf.j2 | sed '/^\s*$/d' > /etc/httpd/conf.d/rucio.conf

echo "=================== /etc/httpd/conf.d/rucio.conf ========================"
cat /etc/httpd/conf.d/rucio.conf
echo ""
useradd rucioacct
mkdir -p /home/rucioacct/.globus/
cp /opt/rucio/x509/usercert.pem /home/rucioacct/.globus/usercert.pem
chmod 644 /home/rucioacct/.globus/usercert.pem
cp /opt/rucio/x509/userkey.pem /home/rucioacct/.globus/userkey.pem
chmod 400 /home/rucioacct/.globus/userkey.pem
chown rucioacct: rucioacct /home/rucioacct/.globus/userkey.pem
chown rucioacct: rucioacct /home/rucioacct/.globus/usercert.pem
yum -y install MySQL-python
rpm -Uvh https://repo.opensciencegrid.org/osg/3.4/osg-3.4-el7-release-latest.rpm
yum install -y osg-ca-certs mod_ssl
yum install -y gfal2-all gfal2-util globus-proxy-utils-7.1-1.el7.x86_64
curl -o setup_data.py -l "https://raw.githubusercontent.com/rucio/rucio/master/etc/docker/demo/setup_data.py"
cp /tmp/httpd/conf.d/rucio.conf /etc/httpd/conf.d/rucio.conf
cp /tmp/rucio-host-pems/hostcert.pem /etc/grid-security/
chmod 644 /etc/grid-security/hostcert.pem
cp /tmp/rucio-host-pems/hostkey.pem /etc/grid-security/
chmod 400 /etc/grid-security/hostkey.pem
chown root:root /etc/grid-security/hostkey.pem
chown root:root /etc/grid-security/hostcert.pem
mkdir -p /tmp/log/rucio/
touch /tmp/log/rucio/httpd_access_log
touch /tmp/log/rucio/httpd_error_log
rm /etc/httpd/conf.d/autoindex.conf
rm /etc/httpd/conf/httpd.conf
cp /tmp/httpdconf/httpd.conf /etc/httpd/conf/httpd.conf

httpd -D FOREGROUND

4) Create the configuration files.

In the `config` directory created at the start of this guide, run:

echo "[common]
logdir = /var/log/rucio
loglevel = DEBUG
mailtemplatedir=/opt/rucio/etc/mail_templates
[client]
rucio_host = https://{Rucio Pod FQDN}:443
auth_host = https://{Rucio Pod FQDN}:443
auth_type = x509
account = root 
ca_cert = /opt/rucio/web/ca.crt
client_cert = /home/rucioacct/.globus/usercert.pem
client_key = /home/rucioacct/.globus/userkey.pem
client_x509_proxy = /tmp/proxycert
request_retries = 3
[database]
default = postgresql://rucio:rucio@127.0.0.1/rucio
pool_reset_on_return = rollback
echo = 0
pool_recycle = 600
[policy]
permission = generic
schema = generic
lfn2pfn_algorithm_default = hash
support = https://github.com/rucio/rucio/issues/
support_rucio = https://github.com/rucio/rucio/issues/
[alembic]
cfg = /opt/rucio/etc/alembic.ini
[conveyor]
scheme = gsiftp
transfertool = fts3
ftshosts = https://{FTS3 FQDN}:8446
usercert = /tmp/proxycert
[messaging-fts3]
brokers = localhost
port = 61616
ssl_key_file = /etc/grid-security/hostkey.pem
ssl_cert_file = /etc/grid-security/hostcert.pem
destination = /topic/transfer.fts_monitoring_queue_state
[policy]
schema = xenon.biggrid.nl
permission = xenon.biggrid.nl" > rucio.cfg
echo "<IfModule !unique_id_module>
  LoadModule unique_id_module modules/mod_unique_id.so
</IfModule>
<IfModule !wsgi_module>
  LoadModule wsgi_module /usr/lib64/httpd/modules/mod_wsgi.so
</IfModule>
<IfModule !authn_core_module>
  LoadModule authn_core_module modules/mod_authn_core.so
</IfModule>
<IfModule !cache_disk_module>
  LoadModule cache_disk_module modules/mod_cache_disk.so
</IfModule>
LoadModule ssl_module /usr/lib64/httpd/modules/mod_ssl.so

<Directory /usr/lib/python2.7>
    AllowOverride none
    Require all granted
</Directory>

#Listen 443
Header set X-Rucio-Host "%{HTTP_HOST}e"
RequestHeader add X-Rucio-RequestId "%{UNIQUE_ID}e"
LogFormat "%h\t%t\t%{X-Rucio-Forwarded-For}i\t%T\t%D\t\"%{X-Rucio-Auth-Token}i\"\t%{X-Rucio-RequestId}i\t%{X-Rucio-Client-Ref}i\t\"%r\"\t%>s\t%b" combinedrucio
CacheEnable disk /
CacheRoot /tmp

<VirtualHost *:443>
  
  ServerName rucio-k8s-test.grid.uchicago.edu
  ServerAdmin manglano@uchicago.edu

  SSLEngine on
  SSLCertificateFile /etc/grid-security/hostcert.pem
  SSLCertificateKeyFile /etc/grid-security/hostkey.pem
  SSLCACertificatePath /etc/grid-security/certificates
  SSLCARevocationPath /etc/grid-security/certificates
  SSLVerifyClient optional
  SSLVerifyDepth 10
  SSLOptions +StdEnvVars
  
  LogLevel debug 
  ErrorLog /tmp/log/rucio/httpd_error_log
  TransferLog /tmp/log/rucio/httpd_access_log

  # these are pretty much all the scripts I could find. Some of them
  # probably are not needed or require additional configuration.
  WSGIScriptAlias /accounts               /usr/lib/python2.7/site-packages/rucio/web/rest/account.py
  WSGIScriptAlias /accountlimits          /usr/lib/python2.7/site-packages/rucio/web/rest/account_limit.py
  WSGIScriptAlias /archives               /usr/lib/python2.7/site-packages/rucio/web/rest/archive.py
  WSGIScriptAlias /auth                   /usr/lib/python2.7/site-packages/rucio/web/rest/authentication.py
  WSGIScriptAlias /config                 /usr/lib/python2.7/site-packages/rucio/web/rest/config.py
  WSGIScriptAlias /dids                   /usr/lib/python2.7/site-packages/rucio/web/rest/did.py
  WSGIScriptAlias /identities             /usr/lib/python2.7/site-packages/rucio/web/rest/identity.py
  WSGIScriptAlias /heartbeats             /usr/lib/python2.7/site-packages/rucio/web/rest/heartbeat.py
  WSGIScriptAlias /lifetime_exceptions    /usr/lib/python2.7/site-packages/rucio/web/rest/lifetime_exception.py
  WSGIScriptAlias /locks                  /usr/lib/python2.7/site-packages/rucio/web/rest/lock.py
  WSGIScriptAlias /meta                   /usr/lib/python2.7/site-packages/rucio/web/rest/meta.py
  WSGIScriptAlias /nongrid_traces         /usr/lib/python2.7/site-packages/rucio/web/rest/nongrid_trace.py
  WSGIScriptAlias /objectstores           /usr/lib/python2.7/site-packages/rucio/web/rest/objectstore.py
  WSGIScriptAlias /ping                   /usr/lib/python2.7/site-packages/rucio/web/rest/ping.py
  WSGIScriptAlias /redirect               /usr/lib/python2.7/site-packages/rucio/web/rest/redirect.py
  WSGIScriptAlias /replicas               /usr/lib/python2.7/site-packages/rucio/web/rest/replica.py
  WSGIScriptAlias /requests               /usr/lib/python2.7/site-packages/rucio/web/rest/request.py
  WSGIScriptAlias /rses                   /usr/lib/python2.7/site-packages/rucio/web/rest/rse.py
  WSGIScriptAlias /rules                  /usr/lib/python2.7/site-packages/rucio/web/rest/rule.py
  WSGIScriptAlias /scopes                 /usr/lib/python2.7/site-packages/rucio/web/rest/scope.py
  WSGIScriptAlias /subscriptions          /usr/lib/python2.7/site-packages/rucio/web/rest/subscription.py
  WSGIScriptAlias /tmp_dids               /usr/lib/python2.7/site-packages/rucio/web/rest/temporary_did.py
  WSGIScriptAlias /traces                 /usr/lib/python2.7/site-packages/rucio/web/rest/trace.py

 <LocationMatch auth/x509>
  SSLVerifyClient optional
  SSLVerifyDepth 10
  SSLOptions +StdEnvVars
 </LocationMatch>
 <LocationMatch /identity/.+/x509>
  SSLVerifyClient require
  SSLVerifyDepth 10
  SSLOptions +StdEnvVars
 </LocationMatch>

</VirtualHost>

# vim:ft=apache" > rucio.conf
echo "# Copyright European Organization for Nuclear Research (CERN)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Vincent Garonne, <vincent.garonne@cern.ch>, 2014

# A generic, single database configuration.

[alembic]
# path to migration scripts
script_location = /usr/lib/python2.7/site-packages/rucio/db/sqla/migrate_repo/

# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s

# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40

# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false

sqlalchemy.url = sqlite:////tmp/rucio.db

# version_table_schema = ATLAS_RUCIO

# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic

[handlers]
keys = console

[formatters]
keys = generic

[logger_root]
level = WARN
handlers = console
qualname =

[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine

[logger_alembic]
level = INFO
handlers =
qualname = alembic

[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic

[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S" > alembic.ini
echo "#
# This is the main Apache HTTP server configuration file.  It contains the
# configuration directives that give the server its instructions.
# See <URL:http://httpd.apache.org/docs/2.4/> for detailed information.
# In particular, see 
# <URL:http://httpd.apache.org/docs/2.4/mod/directives.html>
# for a discussion of each configuration directive.
#
# Do NOT simply read the instructions in here without understanding
# what they do.  They're here only as hints or reminders.  If you are unsure
# consult the online docs. You have been warned.  
#
# Configuration and logfile names: If the filenames you specify for many
# of the server's control files begin with "/" (or "drive:/" for Win32), the
# server will use that explicit path.  If the filenames do *not* begin
# with "/", the value of ServerRoot is prepended -- so 'log/access_log'
# with ServerRoot set to '/www' will be interpreted by the
# server as '/www/log/access_log', where as '/log/access_log' will be
# interpreted as '/log/access_log'.

#
# ServerRoot: The top of the directory tree under which the server's
# configuration, error, and log files are kept.
#
# Do not add a slash at the end of the directory path.  If you point
# ServerRoot at a non-local disk, be sure to specify a local disk on the
# Mutex directive, if file-based mutexes are used.  If you wish to share the
# same ServerRoot for multiple httpd daemons, you will need to change at
# least PidFile.
#
ServerRoot "/etc/httpd"

#
# Listen: Allows you to bind Apache to specific IP addresses and/or
# ports, instead of the default. See also the <VirtualHost>
# directive.
#
# Change this to Listen on specific IP addresses as shown below to 
# prevent Apache from glomming onto all bound IP addresses.
#
#Listen 12.34.56.78:80
#Listen 80 

#
# Dynamic Shared Object (DSO) Support
#
# To be able to use the functionality of a module which was built as a DSO you
# have to place corresponding `LoadModule' lines at this location so the
# directives contained in it are actually available _before_ they are used.
# Statically compiled modules (those listed by `httpd -l') do not need
# to be loaded here.
#
# Example:
# LoadModule foo_module modules/mod_foo.so
#
Include conf.modules.d/*.conf

#
# If you wish httpd to run as a different user or group, you must run
# httpd as root initially and it will switch.  
#
# User/Group: The name (or #number) of the user/group to run httpd as.
# It is usually good practice to create a dedicated user and group for
# running httpd, as with most system services.
#
User apache
Group apache

# 'Main' server configuration
#
# The directives in this section set up the values used by the 'main'
# server, which responds to any requests that aren't handled by a
# <VirtualHost> definition.  These values also provide defaults for
# any <VirtualHost> containers you may define later in the file.
#
# All of these directives may appear inside <VirtualHost> containers,
# in which case these default settings will be overridden for the
# virtual host being defined.
#

#
# ServerAdmin: Your address, where problems with the server should be
# e-mailed.  This address appears on some server-generated pages, such
# as error documents.  e.g. admin@your-domain.com
#
ServerAdmin root@localhost

#
# ServerName gives the name and port that the server uses to identify itself.
# This can often be determined automatically, but we recommend you specify
# it explicitly to prevent problems during startup.
#
# If your host doesn't have a registered DNS name, enter its IP address here.
#
#ServerName www.example.com:80

#
# Deny access to the entirety of your server's filesystem. You must
# explicitly permit access to web content directories in other 
# <Directory> blocks below.
#
<Directory />
    AllowOverride none
    Require all denied
</Directory>

#
# Note that from this point forward you must specifically allow
# particular features to be enabled - so if something's not working as
# you might expect, make sure that you have specifically enabled it
# below.
#

#
# DocumentRoot: The directory out of which you will serve your
# documents. By default, all requests are taken from this directory, but
# symbolic links and aliases may be used to point to other locations.
#
DocumentRoot "/var/www/html"

#
# Relax access to content within /var/www.
#
<Directory "/var/www">
    AllowOverride None
    # Allow open access:
    Require all granted
</Directory>

# Further relax access to the default document root:
<Directory "/var/www/html">
    #
    # Possible values for the Options directive are "None", "All",
    # or any combination of:
    #   Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
    #
    # Note that "MultiViews" must be named *explicitly* --- "Options All"
    # doesn't give it to you.
    #
    # The Options directive is both complicated and important.  Please see
    # http://httpd.apache.org/docs/2.4/mod/core.html#options
    # for more information.
    #
    Options Indexes FollowSymLinks

    #
    # AllowOverride controls what directives may be placed in .htaccess files.
    # It can be "All", "None", or any combination of the keywords:
    #   Options FileInfo AuthConfig Limit
    #
    AllowOverride None

    #
    # Controls who can get stuff from this server.
    #
    Require all granted
</Directory>

#
# DirectoryIndex: sets the file that Apache will serve if a directory
# is requested.
#
<IfModule dir_module>
    DirectoryIndex index.html
</IfModule>

#
# The following lines prevent .htaccess and .htpasswd files from being 
# viewed by Web clients. 
#
<Files ".ht*">
    Require all denied
</Files>

#
# ErrorLog: The location of the error log file.
# If you do not specify an ErrorLog directive within a <VirtualHost>
# container, error messages relating to that virtual host will be
# logged here.  If you *do* define an error logfile for a <VirtualHost>
# container, that host's errors will be logged there and not here.
#
ErrorLog "logs/error_log"

#
# LogLevel: Control the number of messages logged to the error_log.
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
#
LogLevel debug 

<IfModule log_config_module>
    #
    # The following directives define some format nicknames for use with
    # a CustomLog directive (see below).
    #
    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
    LogFormat "%h %l %u %t \"%r\" %>s %b" common

    <IfModule logio_module>
      # You need to enable mod_logio.c to use %I and %O
      LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
    </IfModule>

    #
    # The location and format of the access logfile (Common Logfile Format).
    # If you do not define any access logfiles within a <VirtualHost>
    # container, they will be logged here.  Contrariwise, if you *do*
    # define per-<VirtualHost> access logfiles, transactions will be
    # logged therein and *not* in this file.
    #
    #CustomLog "logs/access_log" common

    #
    # If you prefer a logfile with access, agent, and referer information
    # (Combined Logfile Format) you can use the following directive.
    #
    CustomLog "logs/access_log" combined
</IfModule>

<IfModule alias_module>
    #
    # Redirect: Allows you to tell clients about documents that used to 
    # exist in your server's namespace, but do not anymore. The client 
    # will make a new request for the document at its new location.
    # Example:
    # Redirect permanent /foo http://www.example.com/bar

    #
    # Alias: Maps web paths into filesystem paths and is used to
    # access content that does not live under the DocumentRoot.
    # Example:
    # Alias /webpath /full/filesystem/path
    #
    # If you include a trailing / on /webpath then the server will
    # require it to be present in the URL.  You will also likely
    # need to provide a <Directory> section to allow access to
    # the filesystem path.

    #
    # ScriptAlias: This controls which directories contain server scripts. 
    # ScriptAliases are essentially the same as Aliases, except that
    # documents in the target directory are treated as applications and
    # run by the server when requested rather than as documents sent to the
    # client.  The same rules about trailing "/" apply to ScriptAlias
    # directives as to Alias.
    #
    ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"

</IfModule>

#
# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased
# CGI directory exists, if you have that configured.
#
<Directory "/var/www/cgi-bin">
    AllowOverride None
    Options None
    Require all granted
</Directory>

<IfModule mime_module>
    #
    # TypesConfig points to the file containing the list of mappings from
    # filename extension to MIME-type.
    #
    TypesConfig /etc/mime.types

    #
    # AddType allows you to add to or override the MIME configuration
    # file specified in TypesConfig for specific file types.
    #
    #AddType application/x-gzip .tgz
    #
    # AddEncoding allows you to have certain browsers uncompress
    # information on the fly. Note: Not all browsers support this.
    #
    #AddEncoding x-compress .Z
    #AddEncoding x-gzip .gz .tgz
    #
    # If the AddEncoding directives above are commented-out, then you
    # probably should define those extensions to indicate media types:
    #
    AddType application/x-compress .Z
    AddType application/x-gzip .gz .tgz

    #
    # AddHandler allows you to map certain file extensions to "handlers":
    # actions unrelated to filetype. These can be either built into the server
    # or added with the Action directive (see below)
    #
    # To use CGI scripts outside of ScriptAliased directories:
    # (You will also need to add "ExecCGI" to the "Options" directive.)
    #
    #AddHandler cgi-script .cgi

    # For type maps (negotiated resources):
    #AddHandler type-map var

    #
    # Filters allow you to process content before it is sent to the client.
    #
    # To parse .shtml files for server-side includes (SSI):
    # (You will also need to add "Includes" to the "Options" directive.)
    #
    AddType text/html .shtml
    AddOutputFilter INCLUDES .shtml
</IfModule>

#
# Specify a default charset for all content served; this enables
# interpretation of all content as UTF-8 by default.  To use the 
# default browser choice (ISO-8859-1), or to allow the META tags
# in HTML content to override this choice, comment out this
# directive:
#
AddDefaultCharset UTF-8

<IfModule mime_magic_module>
    #
    # The mod_mime_magic module allows the server to use various hints from the
    # contents of the file itself to determine its type.  The MIMEMagicFile
    # directive tells the module where the hint definitions are located.
    #
    MIMEMagicFile conf/magic
</IfModule>

#
# Customizable error responses come in three flavors:
# 1) plain text 2) local redirects 3) external redirects
#
# Some examples:
#ErrorDocument 500 "The server made a boo boo."
#ErrorDocument 404 /missing.html
#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
#ErrorDocument 402 http://www.example.com/subscription_info.html
#

#
# EnableMMAP and EnableSendfile: On systems that support it, 
# memory-mapping or the sendfile syscall may be used to deliver
# files.  This usually improves server performance, but must
# be turned off when serving from networked-mounted 
# filesystems or if support for these functions is otherwise
# broken on your system.
# Defaults if commented: EnableMMAP On, EnableSendfile Off
#
#EnableMMAP off
EnableSendfile on

# Supplemental configuration
#
# Load config files in the "/etc/httpd/conf.d" directory, if any.
IncludeOptional conf.d/*.conf" > httpd.conf
echo "-----BEGIN CERTIFICATE-----
MIIF+TCCA+GgAwIBAgIQRyDQ+oVGGn4XoWQCkYRjdDANBgkqhkiG9w0BAQwFADCB
iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl
cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV
BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTQx
MDA2MDAwMDAwWhcNMjQxMDA1MjM1OTU5WjB2MQswCQYDVQQGEwJVUzELMAkGA1UE
CBMCTUkxEjAQBgNVBAcTCUFubiBBcmJvcjESMBAGA1UEChMJSW50ZXJuZXQyMREw
DwYDVQQLEwhJbkNvbW1vbjEfMB0GA1UEAxMWSW5Db21tb24gUlNBIFNlcnZlciBD
QTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJwb8bsvf2MYFVFRVA+e
xU5NEFj6MJsXKZDmMwysE1N8VJG06thum4ltuzM+j9INpun5uukNDBqeso7JcC7v
HgV9lestjaKpTbOc5/MZNrun8XzmCB5hJ0R6lvSoNNviQsil2zfVtefkQnI/tBPP
iwckRR6MkYNGuQmm/BijBgLsNI0yZpUn6uGX6Ns1oytW61fo8BBZ321wDGZq0GTl
qKOYMa0dYtX6kuOaQ80tNfvZnjNbRX3EhigsZhLI2w8ZMA0/6fDqSl5AB8f2IHpT
eIFken5FahZv9JNYyWL7KSd9oX8hzudPR9aKVuDjZvjs3YncJowZaDuNi+L7RyML
fzcCAwEAAaOCAW4wggFqMB8GA1UdIwQYMBaAFFN5v1qqK0rPVIDh2JvAnfKyA2bL
MB0GA1UdDgQWBBQeBaN3j2yW4luHS6a0hqxxAAznODAOBgNVHQ8BAf8EBAMCAYYw
EgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
AwIwGwYDVR0gBBQwEjAGBgRVHSAAMAgGBmeBDAECAjBQBgNVHR8ESTBHMEWgQ6BB
hj9odHRwOi8vY3JsLnVzZXJ0cnVzdC5jb20vVVNFUlRydXN0UlNBQ2VydGlmaWNh
dGlvbkF1dGhvcml0eS5jcmwwdgYIKwYBBQUHAQEEajBoMD8GCCsGAQUFBzAChjNo
dHRwOi8vY3J0LnVzZXJ0cnVzdC5jb20vVVNFUlRydXN0UlNBQWRkVHJ1c3RDQS5j
cnQwJQYIKwYBBQUHMAGGGWh0dHA6Ly9vY3NwLnVzZXJ0cnVzdC5jb20wDQYJKoZI
hvcNAQEMBQADggIBAC0RBjjW29dYaK+qOGcXjeIT16MUJNkGE+vrkS/fT2ctyNMU
11ZlUp5uH5gIjppIG8GLWZqjV5vbhvhZQPwZsHURKsISNrqOcooGTie3jVgU0W+0
+Wj8mN2knCVANt69F2YrA394gbGAdJ5fOrQmL2pIhDY0jqco74fzYefbZ/VS29fR
5jBxu4uj1P+5ZImem4Gbj1e4ZEzVBhmO55GFfBjRidj26h1oFBHZ7heDH1Bjzw72
hipu47Gkyfr2NEx3KoCGMLCj3Btx7ASn5Ji8FoU+hCazwOU1VX55mKPU1I2250Lo
RCASN18JyfsD5PVldJbtyrmz9gn/TKbRXTr80U2q5JhyvjhLf4lOJo/UzL5WCXED
Smyj4jWG3R7Z8TED9xNNCxGBMXnMete+3PvzdhssvbORDwBZByogQ9xL2LUZFI/i
eoQp0UM/L8zfP527vWjEzuDN5xwxMnhi+vCToh7J159o5ah29mP+aJnvujbXEnGa
nrNxHzu+AGOePV8hwrGGG7hOIcPDQwkuYwzN/xT29iLp/cqf9ZhEtkGcQcIImH3b
oJ8ifsCnSbu0GB9L06Yqh7lcyvKDTEADslIaeSEINxhO2Y1fmcYFX/Fqrrp1WnhH
OjplXuXE0OPa0utaKC25Aplgom88L2Z8mEWcyfoB7zKOfD759AN7JKZWCYwk
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIFdzCCBF+gAwIBAgIQE+oocFv07O0MNmMJgGFDNjANBgkqhkiG9w0BAQwFADBv
MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk
ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF
eHRlcm5hbCBDQSBSb290MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEwNDgzOFow
gYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpOZXcgSmVyc2V5MRQwEgYDVQQHEwtK
ZXJzZXkgQ2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMS4wLAYD
VQQDEyVVU0VSVHJ1c3QgUlNBIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIICIjAN
BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAgBJlFzYOw9sIs9CsVw127c0n00yt
UINh4qogTQktZAnczomfzD2p7PbPwdzx07HWezcoEStH2jnGvDoZtF+mvX2do2NC
tnbyqTsrkfjib9DsFiCQCT7i6HTJGLSR1GJk23+jBvGIGGqQIjy8/hPwhxR79uQf
jtTkUcYRZ0YIUcuGFFQ/vDP+fmyc/xadGL1RjjWmp2bIcmfbIWax1Jt4A8BQOujM
8Ny8nkz+rwWWNR9XWrf/zvk9tyy29lTdyOcSOk2uTIq3XJq0tyA9yn8iNK5+O2hm
AUTnAU5GU5szYPeUvlM3kHND8zLDU+/bqv50TmnHa4xgk97Exwzf4TKuzJM7UXiV
Z4vuPVb+DNBpDxsP8yUmazNt925H+nND5X4OpWaxKXwyhGNVicQNwZNUMBkTrNN9
N6frXTpsNVzbQdcS2qlJC9/YgIoJk2KOtWbPJYjNhLixP6Q5D9kCnusSTJV882sF
qV4Wg8y4Z+LoE53MW4LTTLPtW//e5XOsIzstAL81VXQJSdhJWBp/kjbmUZIO8yZ9
HE0XvMnsQybQv0FfQKlERPSZ51eHnlAfV1SoPv10Yy+xUGUJ5lhCLkMaTLTwJUdZ
+gQek9QmRkpQgbLevni3/GcV4clXhB4PY9bpYrrWX1Uu6lzGKAgEJTm4Diup8kyX
HAc/DVL17e8vgg8CAwEAAaOB9DCB8TAfBgNVHSMEGDAWgBStvZh6NLQm9/rEJlTv
A73gJMtUGjAdBgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/
BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wEQYDVR0gBAowCDAGBgRVHSAAMEQGA1Ud
HwQ9MDswOaA3oDWGM2h0dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9BZGRUcnVzdEV4
dGVybmFsQ0FSb290LmNybDA1BggrBgEFBQcBAQQpMCcwJQYIKwYBBQUHMAGGGWh0
dHA6Ly9vY3NwLnVzZXJ0cnVzdC5jb20wDQYJKoZIhvcNAQEMBQADggEBAJNl9jeD
lQ9ew4IcH9Z35zyKwKoJ8OkLJvHgwmp1ocd5yblSYMgpEg7wrQPWCcR23+WmgZWn
RtqCV6mVksW2jwMibDN3wXsyF24HzloUQToFJBv2FAY7qCUkDrvMKnXduXBBP3zQ
YzYhBx9G/2CkkeFnvN4ffhkUyWNnkepnB2u0j4vAbkN9w6GAbLIevFOFfdyQoaS8
Le9Gclc1Bb+7RrtubTeZtv8jkpHGbkD4jylW6l/VXxRTrPBPYer3IsynVgviuDQf
Jtl7GQVoP7o81DgGotPmjw7jtHFtQELFhLRAlSv0ZaBIefYdgWOWnU914Ph85I6p
0fKtirOMxyHNwu8=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEU
MBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFs
IFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290
MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEwNDgzOFowbzELMAkGA1UEBhMCU0Ux
FDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRUcnVzdCBFeHRlcm5h
bCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0EgUm9v
dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvt
H7xsD821+iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9
uMq/NzgtHj6RQa1wVsfwTz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzX
mk6vBbOmcZSccbNQYArHE504B4YCqOmoaSYYkKtMsE8jqzpPhNjfzp/haW+710LX
a0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy2xSoRcRdKn23tNbE7qzN
E0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv77+ldU9U0
WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYD
VR0PBAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0
Jvf6xCZU7wO94CTLVBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRU
cnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsx
IjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENBIFJvb3SCAQEwDQYJKoZIhvcN
AQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZlj7DYd7usQWxH
YINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5
6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvC
Nr4TDea9Y355e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEX
c4g/VhsxOBi0cQ+azcgOno4uG+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5a
mnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ=
-----END CERTIFICATE-----" > ca.crt
echo "all:
   -kubectl delete configmap rucio-test-etc
   kubectl create configmap rucio-test-etc --from-file rucio.cfg --from-file alembic.ini
   -kubectl delete configmap rucio-test-httpd
   kubectl create configmap rucio-test-httpd --from-file rucio.conf
   -kubectl delete configmap rucio-httpd-conf
   kubectl create configmap rucio-httpd-conf --from-file httpd.conf
   -kubectl delete configmap rucio-entrypoint
   kubectl create configmap rucio-entrypoint --from-file docker-entrypoint.sh
   -kubectl delete configmap rucio-ca-crt
   kubectl create configmap rucio-ca-crt --from-file ca.crt" > Makefile

5) Get a user X509 identity.

Follow the instructions at http://twiki.mwt2.org/bin/view/Main/RucioNewUserStartGuide#Getting_a_user_certificate to obtain and download the X509 identity p12 for your identity. After extracting the PEM files, move them to the folder `~/Code/Kubernetes/Rucio/` and name the certificate `usercert.pem` and the key `userkey.pem`.

6) Get a hostname SSL identity.

Follow the instructions at http://twiki.mwt2.org/bin/view/Main/RucioVMAdminGuide#IGTF_Certification to generate a key, and obtain a certificate, for the SSL hostname identity. Move the key to `~/Code/Kubernetes/Rucio/` as `hostkey.pem` and move the certificate to the folder as `hostcert.pem`.

7) Populate Kubernetes secrets with the X509 & hostname.

cd ~/Code/Kubernetes/Rucio/
kubectl create secret generic rucio-host-pems --from-file=hostcert.pem --from-file=hostkey.pem
kubectl create secret generic rucio-test-x509 --from-file=usercert.pem --from-file=userkey.pem

8) Populate the configmap with configuration files.

cd config && make && cd ..

9) Launch the deployment.

kubectl apply -f deployment.yaml

GridFTP Pods

1) Create a directory for the Kubernetes files.

mkdir -p ~/Code/Kubernetes/GridFTP/config && cd ~/Code/Kubernetes/GridFTP

2) Create the deployment.yaml.

`vi ~/Code/Kubernetes/GridFTP` and paste:

echo "apiVersion: extensions/v1beta1
kind: Deployment 
metadata:
  name: rucio-gridftp
  labels:
    app: rucio-gridftp
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: rucio-gridftp
    spec:
      hostNetwork: true
      hostname: ruciogridftp2
      volumes:
        - name: gridftp-persistent-storage
          persistentVolumeClaim:
            claimName: gridftp-persistent-claim
        - name: gridftp-test-x509
          secret:
            secretName: gridftp-test-x509
            defaultMode: 256
        - name: gridftp-entrypoint
          configMap:
            name: gridftp-entrypoint 
        - name: gridftp-host-pems
          secret:
            secretName: gridftp-host-pems
            defaultMode: 256
      containers:
        - name: rucio-gridftp
          image: davidmanglano/gridftp 
          command: ["/bin/sh/"]
          args: ["/tmp/entrypoint/docker-entrypoint.sh/"]
          volumeMounts:
            - name: rucio-test-x509
              mountPath: /opt/rucio/x509
            - name: gridftp-entrypoint
              mountPath: /tmp/entrypoint/
            - name: gridftp-persistent-storage
              mountPath: /scratch
            - name: gridftp-host-pems
              mountPath: /tmp/gridftp-host-pems/
      nodeSelector:
        gridftp: "true"

3) Create the entrypoint shell script.

echo "#!/bin/bash -e
  
cp /opt/rucio/x509/usercert.pem /home/gridftp/.globus/usercert.pem
chmod 644 /home/gridftp/.globus/usercert.pem
cp /opt/rucio/x509/userkey.pem /home/gridftp/.globus/userkey.pem
chmod 400 /home/gridftp/.globus/userkey.pem
chown gridftp:gridftp /home/gridftp/.globus/userkey.pem
chown gridftp:gridftp /home/gridftp/.globus/usercert.pem
chown gridftp:gridftp /scratch
cp /tmp/gridftp-host-pems/hostcert.pem /etc/grid-security/
chmod 644 /etc/grid-security/hostcert.pem
chown root:root /etc/grid-security/hostcert.pem
cp /tmp/gridftp-host-pems/hostkey.pem /etc/grid-security/
chmod 400 /etc/grid-security/hostkey.pem
chown root:root /etc/grid-security/hostkey.pem
echo -e '"/DC=org/DC=cilogon/C=US/O=University of Chicago/CN=David Manglano A3121466" gridftp' > /etc/grid-security/grid-mapfile
globus-gridftp-server" > docker-entrypoint.sh

4) Get a user X509 identity.

Follow the instructions at http://twiki.mwt2.org/bin/view/Main/RucioNewUserStartGuide#Getting_a_user_certificate to obtain and download the X509 identity p12 for your identity. After extracting the PEM files, move them to the folder `~/Code/Kubernetes/GridFTP/` and name the certificate `usercert.pem` and the key `userkey.pem`.

5) Get a hostname SSL identity.

Follow the instructions at http://twiki.mwt2.org/bin/view/Main/RucioVMAdminGuide#IGTF_Certification to generate a key, and obtain a certificate, for the SSL hostname identity. Move the key to `~/Code/Kubernetes/GridFTP/` as `hostkey.pem` and move the certificate to the folder as `hostcert.pem`.

6) Populate Kubernetes secrets with the X509 & hostname identities.

cd ~/Code/Kubernetes/GridFTP/
kubectl create secret generic gridftp-host-pems --from-file=hostcert.pem --from-file=hostkey.pem
kubectl create secret generic gridftp-test-x509 --from-file=usercert.pem --from-file=userkey.pem

7) Populate the configmap with configuration files.

kubectl create configmap gridftp-entrypoint --from-file gridftp/docker-entrypoint.sh

8) Launch the deployment.

kubectl apply -f deployment.yaml

File Transfer Service 3 Pods

1) Create a directory for the Kubernetes files.

mkdir -p ~/Code/Kubernetes/FTS3/config && cd ~/Code/Kubernetes/GridFTP

2) Create a deployment.yaml.

`vi deployment.yaml` and paste:

apiVersion: extensions/v1beta1
kind: Deployment 
metadata:
  name: rucio-fts3
  labels:
    app: rucio-fts3
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: rucio-fts3
    spec:
      hostname: rucio-fts3
      volumes:
      - name: rucio-test-x509
        secret:
          secretName: rucio-test-x509
          defaultMode: 256
      - name: fts3-entrypoint
        configMap:
          name: fts3-entrypoint 
      - name: fts3-host-pems
        secret:
          secretName: fts3-host-pems
          defaultMode: 256
      - name: mariadb-persistent-storage
        persistentVolumeClaim:
          claimName: mariadb-persistent-claim
      containers:
      - name: rucio-fts3
        image: davidmanglano/fts3
        command: ["/bin/sh"]
        args: ["/tmp/entrypoint/docker-entrypoint.sh"]
        volumeMounts:
        - name: rucio-test-x509
          mountPath: /opt/rucio/x509
        - name: fts3-entrypoint
          mountPath: /tmp/entrypoint/
        - name: fts3-host-pems
          mountPath: /tmp/fts3-host-pems/
        ports:
        - containerPort: 8846
          name: fts3-rest
        - containerPort: 8849
          name: fts3-web
      - name: mariadb
        image: mariadb:10.3.15
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: "FTS3"
        - name: MYSQL_USER
          value: "FTS3"
        - name: MYSQL_PASSWORD
          value: "FTS3"
        - name: MYSQL_DATABASE
          value: "FTS3"
        ports:
        - containerPort: 3306
          name: mariadb
        volumeMounts:
        - name: mariadb-persistent-storage
          mountPath: /var/lib/mysql
---
apiVersion: v1
kind: Service
metadata:
  name: rucio-fts3
  labels:
    app: rucio-fts3
spec:
  type: NodePort 
  ports:
    - port: 8446 
      name: rest
      protocol: TCP
      targetPort: 8446 
    - port: 8449
      name: web
      protocol: TCP
      targetPort: 8449
  selector:
    app: rucio-fts3

3) Create the entrypoint shell script.

#!/bin/bash -e
cp /tmp/fts3-host-pems/hostcert.pem /etc/grid-security/
chmod 644 /etc/grid-security/hostcert.pem
chown root:root /etc/grid-security/hostcert.pem
cp /tmp/fts3-host-pems/hostkey.pem /etc/grid-security/
chmod 400 /etc/grid-security/hostkey.pem
chown root:root /etc/grid-security/hostkey.pem
supervisord -c /etc/supervisor/conf.d/supervisord.conf

4) Get a hostname SSL identity.

Follow the instructions at http://twiki.mwt2.org/bin/view/Main/RucioVMAdminGuide#IGTF_Certification to generate a key, and obtain a certificate, for the SSL hostname identity. Move the key to `~/Code/Kubernetes/FTS3/` as `hostkey.pem` and move the certificate to the folder as `hostcert.pem`.

5) Populate secrets with the hostname identities.

kubectl create secret generic fts3-host-pems --from-file=hostcert.pem --from-file=hostkey.pem

6) Populate the configmap with configuration files.

kubectl create configmap fts3-entrypoint --from-file fts3/docker-entrypoint.sh

7) Launch the deployment.

kubectl apply -f deployment.yaml

ActiveMQ Pods

1) Create a directory for the Kubernetes files.

mkdir -p ~/Code/Kubernetes/ActiveMQ/config && cd ~/Code/Kubernetes/ActiveMQ

2) Create a deployment.yaml.

`vi deployment.yaml` and paste:

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: rucio-server
  labels:
    app: rucio-server
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: rucio-server
    spec:
      containers:
        name: rucio-activemq 
        image: davidmanglano/activemq 
        imagePullPolicy: Always
        ports:
          - containerPort: 61616 
        command: ["/bin/sh"]
        args: ["./~/apache-activemq-5.15.9-bin/bin/activemq console"]
---
apiVersion: v1
kind: Service
metadata:
  name: rucio-activemq
  labels:
    app: rucio-activemq
  annotations:
    external-dns.alpha.kubernetes.io/hostname: kubernetes-activemq.grid.uchicago.edu
spec:
  type: LoadBalancer 
  ports:
    - port: 61616
      protocol: TCP
      targetPort: 61616
  selector:
    app: rucio-activemq

3) Launch the deployment.

kubectl apply -f deployment.yaml

Prometheus and Grafana Pods

1) Create a directory for the Kubernetes files.

mkdir -p ~/Code/Kubernetes/PrometheusGrafana/config && cd ~/Code/Kubernetes/PrometheusGrafana

2) Create a deployment.yaml.

3) Create the entrypoint shell script.

4) Create the configuration files.

5) Create a Kubernetes secret for the SSL identity.

6) Populate the configmap with configuration files.

Synchronize the pods and validate the server and infrastructure instances.

-- DavidManglano - 22 May 2019</verbatim>