Because my previous installation ran out of space when trying to restore the awx database I've reinstalled Ubuntu Server as Open SSH server with microk8s installed.
No I am unable to get awx up and running and am not sure why.
Add --allow-privileged into /var/snap/microk8s/current/args/kube-apiserver followed by microk8s.stop and microk8s start.
Postgres log shows and google searches doesn't really show any usefull information about this error.
The postgres files are present in /var/snap/microk8s/common/default-storage/awx-postgres-awx-postgres-0-pvc-e1ca6588-3780-41bb-820e-2321f6e60e1c/data
ansible-playbook 2.9.6
config file = /etc/ansible/ansible.cfg
configured module search path = ['/usr/share/ansible/openshift']
ansible python module location = /usr/local/lib/python3.6/site-packages/ansible
executable location = /usr/local/bin/ansible-playbook
python version = 3.6.8 (default, Oct 11 2019, 15:04:54) [GCC 8.3.1 20190507 (Red Hat 8.3.1-4)]
Using /etc/ansible/ansible.cfg as config file
PLAYBOOK: main.yml *************************************************************
1 plays in /opt/ansible/main.yml
PLAY [localhost] ***************************************************************
META: ran handlers
TASK [tower : Ensure configured Tower resources exist in the cluster.] *********
task path: /opt/ansible/roles/tower/tasks/main.yml:2
changed: [localhost] => (item=tower_memcached.yaml.j2) => {"ansible_loop_var": "item", "changed": true, "item": "tower_memcached.yaml.j2", "result": {"results": [{"changed": true, "method": "create", "result": {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"creationTimestamp": "2020-03-19T08:26:31Z", "generation": 1, "labels": {"app": "tower-memcached"}, "name": "awx-memcached", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5630", "selfLink": "/apis/apps/v1/namespaces/awx/deployments/awx-memcached", "uid": "4117354d-2673-454c-959b-b06f917726dc"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 1, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "tower-memcached"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"creationTimestamp": null, "labels": {"app": "tower-memcached"}}, "spec": {"containers": [{"image": "memcached:alpine", "imagePullPolicy": "IfNotPresent", "name": "memcached", "ports": [{"containerPort": 11211, "protocol": "TCP"}], "resources": {}, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File"}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "terminationGracePeriodSeconds": 30}}}, "status": {}}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "v1", "kind": "Service", "metadata": {"creationTimestamp": "2020-03-19T08:26:31Z", "labels": {"app": "tower-memcached"}, "name": "awx-memcached", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5643", "selfLink": "/api/v1/namespaces/awx/services/awx-memcached", "uid": "9898bcda-a845-4013-bb17-6535d8aac57b"}, "spec": {"clusterIP": "None", "ports": [{"port": 11211, "protocol": "TCP", "targetPort": 11211}], "selector": {"app": "tower-memcached"}, "sessionAffinity": "None", "type": "ClusterIP"}, "status": {"loadBalancer": {}}}, "warnings": []}]}}
changed: [localhost] => (item=tower_postgres.yaml.j2) => {"ansible_loop_var": "item", "changed": true, "item": "tower_postgres.yaml.j2", "result": {"results": [{"changed": true, "method": "create", "result": {"apiVersion": "v1", "data": {"password": "YXd4cGFzczk="}, "kind": "Secret", "metadata": {"creationTimestamp": "2020-03-19T08:26:33Z", "name": "awx-postgres-pass", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5649", "selfLink": "/api/v1/namespaces/awx/secrets/awx-postgres-pass", "uid": "67042d2a-41a1-486a-97d7-525164ef8dc1"}, "type": "Opaque"}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "apps/v1", "kind": "StatefulSet", "metadata": {"creationTimestamp": "2020-03-19T08:26:33Z", "generation": 1, "labels": {"app": "tower-postgres"}, "name": "awx-postgres", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5650", "selfLink": "/apis/apps/v1/namespaces/awx/statefulsets/awx-postgres", "uid": "28befb75-ee99-4f78-9894-38a90f14d29f"}, "spec": {"podManagementPolicy": "OrderedReady", "replicas": 1, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "tower-postgres"}}, "serviceName": "awx", "template": {"metadata": {"creationTimestamp": null, "labels": {"app": "tower-postgres"}}, "spec": {"containers": [{"env": [{"name": "POSTGRES_DB", "value": "awx"}, {"name": "POSTGRES_USER", "value": "awx"}, {"name": "POSTGRES_PASSWORD", "valueFrom": {"secretKeyRef": {"key": "password", "name": "awx-postgres-pass"}}}], "image": "postgres:10", "imagePullPolicy": "IfNotPresent", "name": "postgres", "ports": [{"containerPort": 3306, "name": "postgres", "protocol": "TCP"}], "resources": {}, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [{"mountPath": "/var/lib/postgresql/data", "name": "postgres", "subPath": "data"}]}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "terminationGracePeriodSeconds": 30}}, "updateStrategy": {"type": "RollingUpdate"}, "volumeClaimTemplates": [{"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"creationTimestamp": null, "name": "postgres"}, "spec": {"accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "12Gi"}}, "volumeMode": "Filesystem"}, "status": {"phase": "Pending"}}]}, "status": {"replicas": 0}}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "v1", "kind": "Service", "metadata": {"creationTimestamp": "2020-03-19T08:26:33Z", "labels": {"app": "tower-postgres"}, "name": "awx-postgres", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5658", "selfLink": "/api/v1/namespaces/awx/services/awx-postgres", "uid": "a0c286ca-6585-4036-9647-91b3f3264060"}, "spec": {"clusterIP": "None", "ports": [{"port": 5432, "protocol": "TCP", "targetPort": 5432}], "selector": {"app": "tower-postgres"}, "sessionAffinity": "None", "type": "ClusterIP"}, "status": {"loadBalancer": {}}}, "warnings": []}]}}
changed: [localhost] => (item=tower_rabbitmq.yaml.j2) => {"ansible_loop_var": "item", "changed": true, "item": "tower_rabbitmq.yaml.j2", "result": {"results": [{"changed": true, "method": "create", "result": {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"creationTimestamp": "2020-03-19T08:26:34Z", "generation": 1, "labels": {"app": "tower-rabbitmq"}, "name": "awx-rabbitmq", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5684", "selfLink": "/apis/apps/v1/namespaces/awx/deployments/awx-rabbitmq", "uid": "546e6ade-3f39-43e6-9a5d-c8b2c1162e68"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 1, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "tower-rabbitmq"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"creationTimestamp": null, "labels": {"app": "tower-rabbitmq"}}, "spec": {"containers": [{"env": [{"name": "RABBITMQ_DEFAULT_VHOST", "value": "awx"}, {"name": "RABBITMQ_NODE_PORT", "value": "5672"}], "image": "rabbitmq:3", "imagePullPolicy": "IfNotPresent", "name": "rabbitmq", "ports": [{"containerPort": 15672, "protocol": "TCP"}, {"containerPort": 5672, "protocol": "TCP"}], "resources": {}, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File"}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "terminationGracePeriodSeconds": 30}}}, "status": {}}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "v1", "kind": "Service", "metadata": {"creationTimestamp": "2020-03-19T08:26:34Z", "labels": {"app": "tower-rabbitmq"}, "name": "awx-rabbitmq", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5689", "selfLink": "/api/v1/namespaces/awx/services/awx-rabbitmq", "uid": "ffd04c66-5789-4c92-a205-97b6a57656b2"}, "spec": {"clusterIP": "None", "ports": [{"port": 5672, "protocol": "TCP", "targetPort": 5672}], "selector": {"app": "tower-rabbitmq"}, "sessionAffinity": "None", "type": "ClusterIP"}, "status": {"loadBalancer": {}}}, "warnings": []}]}}
y \"default-src 'self'; connect-src 'self' ws: wss:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' *.pendo.io; img-src 'self' *.pendo.io data:; report-uri /csp-violation/\";\n\n # Protect against click-jacking https://www.owasp.org/index.php/Testing_for_Clickjacking_(OTG-CLIENT-009)\n add_header X-Frame-Options \"DENY\";\n\n location /nginx_status {\n stub_status on;\n access_log off;\n allow 127.0.0.1;\n deny all;\n }\n\n location /static/ {\n alias /var/lib/awx/public/static/;\n }\n\n location /favicon.ico {\n alias /var/lib/awx/public/static/favicon.ico;\n }\n\n location /websocket {\n # Pass request to the upstream alias\n proxy_pass http://daphne;\n # Require http version 1.1 to allow for upgrade requests\n proxy_http_version 1.1;\n # We want proxy_buffering off for proxying to websockets.\n proxy_buffering off;\n # http://en.wikipedia.org/wiki/X-Forwarded-For\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n # enable this if you use HTTPS:\n proxy_set_header X-Forwarded-Proto https;\n # pass the Host: header from the client for the sake of redirects\n proxy_set_header Host $http_host;\n # We've set the Host header, so we don't need Nginx to muddle\n # about with redirects\n proxy_redirect off;\n # Depending on the request value, set the Upgrade and\n # connection headers\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection $connection_upgrade;\n }\n\n location / {\n # Add trailing / if missing\n rewrite ^(.*)$http_host(.*[^/])$ $1$http_host$2/ permanent;\n uwsgi_read_timeout 120s;\n uwsgi_pass uwsgi;\n include /etc/nginx/uwsgi_params; proxy_set_header X-Forwarded-Port 443;\n }\n }\n}\n", "settings": "import os\nimport socket\n\ndef get_secret():\n if os.path.exists(\"/etc/tower/SECRET_KEY\"):\n return open('/etc/tower/SECRET_KEY', 'rb').read().strip()\n\nADMINS = ()\nSTATIC_ROOT = '/var/lib/awx/public/static'\nPROJECTS_ROOT = '/var/lib/awx/projects'\nJOBOUTPUT_ROOT = '/var/lib/awx/job_status'\n\nSECRET_KEY = get_secret()\n\nALLOWED_HOSTS = ['*']\n\nINTERNAL_API_URL = 'http://127.0.0.1:8052'\n\n# Container environments don't like chroots\nAWX_PROOT_ENABLED = False\n\n# Automatically deprovision pods that go offline\nAWX_AUTO_DEPROVISION_INSTANCES = True\n\nCLUSTER_HOST_ID = socket.gethostname()\nSYSTEM_UUID = '00000000-0000-0000-0000-000000000000'\n\nCSRF_COOKIE_SECURE = False\nSESSION_COOKIE_SECURE = False\n\nSERVER_EMAIL = 'root@localhost'\nDEFAULT_FROM_EMAIL = 'webmaster@localhost'\nEMAIL_SUBJECT_PREFIX = '[AWX] '\n\nEMAIL_HOST = 'localhost'\nEMAIL_PORT = 25\nEMAIL_HOST_USER = ''\nEMAIL_HOST_PASSWORD = ''\nEMAIL_USE_TLS = False\n\nLOGGING['handlers']['console'] = {\n '()': 'logging.StreamHandler',\n 'level': 'DEBUG',\n 'formatter': 'simple',\n}\n\nLOGGING['loggers']['django.request']['handlers'] = ['console']\nLOGGING['loggers']['rest_framework.request']['handlers'] = ['console']\nLOGGING['loggers']['awx']['handlers'] = ['console', 'external_logger']\nLOGGING['loggers']['awx.main.commands.run_callback_receiver']['handlers'] = ['console']\nLOGGING['loggers']['awx.main.tasks']['handlers'] = ['console', 'external_logger']\nLOGGING['loggers']['awx.main.scheduler']['handlers'] = ['console', 'external_logger']\nLOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']\nLOGGING['loggers']['social']['handlers'] = ['console']\nLOGGING['loggers']['system_tracking_migrations']['handlers'] = ['console']\nLOGGING['loggers']['rbac_migrations']['handlers'] = ['console']\nLOGGING['loggers']['awx.isolated.manager.playbooks']['handlers'] = ['console']\nLOGGING['handlers']['callback_receiver'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['task_system'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['tower_warnings'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['rbac_migrations'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['system_tracking_migrations'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['management_playbooks'] = {'class': 'logging.NullHandler'}\n\nDATABASES = {\n 'default': {\n 'ATOMIC_REQUESTS': True,\n 'ENGINE': 'awx.main.db.profiled_pg',\n 'NAME': 'awx',\n 'USER': 'awx',\n 'PASSWORD': 'awxpass9',\n 'HOST': 'awx-postgres.awx.svc.cluster.local',\n 'PORT': '5432',\n }\n}\n\nif os.getenv(\"DATABASE_SSLMODE\", False):\n DATABASES['default']['OPTIONS'] = {'sslmode': os.getenv(\"DATABASE_SSLMODE\")}\n\nCACHES = {\n 'default': {\n 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',\n 'LOCATION': '{}:{}'.format(\"awx-memcached.awx.svc.cluster.local\", \"11211\")\n },\n 'ephemeral': {\n 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',\n },\n}\n\nBROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format(\n 'guest',\n 'guest',\n 'awx-rabbitmq.awx.svc.cluster.local',\n '5672',\n 'awx')\n\nCHANNEL_LAYERS = {\n 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',\n 'ROUTING': 'awx.main.routing.channel_routing',\n 'CONFIG': {'url': BROKER_URL}}\n}\n\nUSE_X_FORWARDED_PORT = True\n"}, "kind": "ConfigMap", "metadata": {"creationTimestamp": "2020-03-19T08:26:35Z", "labels": {"app": "tower"}, "name": "awx-tower-configmap", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5710", "selfLink": "/api/v1/namespaces/awx/configmaps/awx-tower-configmap", "uid": "7d37fc8d-940a-4changed: [localhost] => (item=tower_config.yaml.j2) => {"ansible_loop_var": "item", "changed": true, "item": "tower_config.yaml.j2", "method": "create", "result": {"apiVersion": "v1", "data": {"environment": "DATABASE_USER=awx\nDATABASE_NAME=awx\nDATABASE_HOST='awx-postgres.awx.svc.cluster.local'\nDATABASE_PORT='5432'\nDATABASE_PASSWORD=awxpass9\nMEMCACHED_HOST='awx-memcached.awx.svc.cluster.local'\nMEMCACHED_PORT='11211'\nRABBITMQ_HOST='awx-rabbitmq.awx.svc.cluster.local'\nRABBITMQ_PORT='5672'\nAWX_SKIP_MIGRATIONS=true\n", "nginx_conf": "worker_processes 1;\npid /tmp/nginx.pid;\n\nevents {\n worker_connections 1024;\n}\n\nhttp {\n include /etc/nginx/mime.types;\n default_type application/octet-stream;\n server_tokens off;\n\n log_format main '$remote_addr - $remote_user [$time_local] \"$request\" '\n '$status $body_bytes_sent \"$http_referer\" '\n '\"$http_user_agent\" \"$http_x_forwarded_for\"';\n\n access_log /dev/stdout main;\n\n map $http_upgrade $connection_upgrade {\n default upgrade;\n '' close;\n }\n\n sendfile on;\n #tcp_nopush on;\n #gzip on;\n\n upstream uwsgi {\n server 127.0.0.1:8050;\n }\n\n upstream daphne {\n server 127.0.0.1:8051;\n }\n\n server {\n listen 8052 default_server;\n\n # If you have a domain name, this is where to add it\n server_name _;\n keepalive_timeout 65;\n\n # HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)\n add_header Strict-Transport-Security max-age=15768000;\n add_header Content-Security-Policy \"default-src 'self'; connect-src 'self' ws: wss:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' *.pendo.io; img-src 'self' *.pendo.io data:; report-uri /csp-violation/\";\n add_header X-Content-Security-Policy \"default-src 'self'; connect-src 'self' ws: wss:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' *.pendo.io; img-src 'self' *.pendo.io data:; report-uri /csp-violation/\";\n\n # Protect against click-jacking https://www.owasp.org/index.php/Testing_for_Clickjacking_(OTG-CLIENT-009)\n add_header X-Frame-Options \"DENY\";\n\n location /nginx_status {\n stub_status on;\n access_log off;\n allow 127.0.0.1;\n deny all;\n }\n\n location /static/ {\n alias /var/lib/awx/public/static/;\n }\n\n location /favicon.ico {\n alias /var/lib/awx/public/static/favicon.ico;\n }\n\n location /websocket {\n # Pass request to the upstream alias\n proxy_pass http://daphne;\n # Require http version 1.1 to allow for upgrade requests\n proxy_http_version 1.1;\n # We want proxy_buffering off for proxying to websockets.\n proxy_buffering off;\n # http://en.wikipedia.org/wiki/X-Forwarded-For\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n # enable this if you use HTTPS:\n proxy_set_header X-Forwarded-Proto https;\n # pass the Host: header from the client for the sake of redirects\n proxy_set_header Host $http_host;\n # We've set the Host header, so we don't need Nginx to muddle\n # about with redirects\n proxy_redirect off;\n # Depending on the request value, set the Upgrade and\n # connection headers\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection $connection_upgrade;\n }\n\n location / {\n # Add trailing / if missing\n rewrite ^(.*)$http_host(.*[^/])$ $1$http_host$2/ permanent;\n uwsgi_read_timeout 120s;\n uwsgi_pass uwsgi;\n include /etc/nginx/uwsgi_params; proxy_set_header X-Forwarded-Port 443;\n }\n }\n}\n", "settings": "import os\nimport socket\n\ndef get_secret():\n if os.path.exists(\"/etc/tower/SECRET_KEY\"):\n return open('/etc/tower/SECRET_KEY', 'rb').read().strip()\n\nADMINS = ()\nSTATIC_ROOT = '/var/lib/awx/public/static'\nPROJECTS_ROOT = '/var/lib/awx/projects'\nJOBOUTPUT_ROOT = '/var/lib/awx/job_status'\n\nSECRET_KEY = get_secret()\n\nALLOWED_HOSTS = ['*']\n\nINTERNAL_API_URL = 'http://127.0.0.1:8052'\n\n# Container environments don't like chroots\nAWX_PROOT_ENABLED = False\n\n# Automatically deprovision pods that go offline\nAWX_AUTO_DEPROVISION_INSTANCES = True\n\nCLUSTER_HOST_ID = socket.gethostname()\nSYSTEM_UUID = '00000000-0000-0000-0000-000000000000'\n\nCSRF_COOKIE_SECURE = False\nSESSION_COOKIE_SECURE = False\n\nSERVER_EMAIL = 'root@localhost'\nDEFAULT_FROM_EMAIL = 'webmaster@localhost'\nEMAIL_SUBJECT_PREFIX = '[AWX] '\n\nEMAIL_HOST = 'localhost'\nEMAIL_PORT = 25\nEMAIL_HOST_USER = ''\nEMAIL_HOST_PASSWORD = ''\nEMAIL_USE_TLS = False\n\nLOGGING['handlers']['console'] = {\n '()': 'logging.StreamHandler',\n 'level': 'DEBUG',\n 'formatter': 'simple',\n}\n\nLOGGING['loggers']['django.request']['handlers'] = ['console']\nLOGGING['loggers']['rest_framework.request']['handlers'] = ['console']\nLOGGING['loggers']['awx']['handlers'] = ['console', 'external_logger']\nLOGGING['loggers']['awx.main.commands.run_callback_receiver']['handlers'] = ['console']\nLOGGING['loggers']['awx.main.tasks']['handlers'] = ['console', 'external_logger']\nLOGGING['loggers']['awx.main.scheduler']['handlers'] = ['console', 'external_logger']\nLOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']\nLOGGING['loggers']['social']['handlers'] = ['console']\nLOGGING['loggers']['system_tracking_migrations']['handlers'] = ['console']\nLOGGING['loggers']['rbac_migrations']['handlers'] = ['console']\nLOGGING['loggers']['awx.isolated.manager.playbooks']['handlers'] = ['console']\nLOGGING['handlers']['callback_receiver'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['task_system'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['tower_warnings'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['rbac_migrations'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['system_tracking_migrations'] = {'class': 'logging.NullHandler'}\nLOGGING['handlers']['management_playbooks'] = {'class': 'logging.NullHandler'}\n\nDATABASES = {\n 'default': {\n 'ATOMIC_REQUESTS': True,\n 'ENGINE': 'awx.main.db.profiled_pg',\n 'NAME': 'awx',\n 'USER': 'awx',\n 'PASSWORD': 'awxpass9',\n 'HOST': 'awx-postgres.awx.svc.cluster.local',\n 'PORT': '5432',\n }\n}\n\nif os.getenv(\"DATABASE_SSLMODE\", False):\n DATABASES['default']['OPTIONS'] = {'sslmode': os.getenv(\"DATABASE_SSLMODE\")}\n\nCACHES = {\n 'default': {\n 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',\n 'LOCATION': '{}:{}'.format(\"awx-memcached.awx.svc.cluster.local\", \"11211\")\n },\n 'ephemeral': {\n 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',\n },\n}\n\nBROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format(\n 'guest',\n 'guest',\n 'awx-rabbitmq.awx.svc.cluster.local',\n '5672',\n 'awx')\n\nCHANNEL_LAYERS = {\n 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',\n 'ROUTING': 'awx.main.routing.channel_routing',\n 'CONFIG': {'url': BROKER_URL}}\n}\n\nUSE_X_FORWARDED_PORT = True\n"}, "kind": "ConfigMap", "metadata": {"creationTimestamp": "2020-03-19T08:26:35Z", "labels": {"app": "tower"}, "name": "awx-tower-configmap", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5710", "selfLink": "/api/v1/namespaces/awx/configmaps/awx-tower-configmap", "uid": "7d37fc8d-940a-4907-8de7-d20517f103a1"}}}
": "/etc/tower/SECRET_KEY", "name": "secret-key", "readOnly": true, "subPath": "SECRET_KEY"}, {"mountPath": "/etc/tower/conf.d/environment.sh", "name": "environment", "readOnly": true, "subPath": "environment.sh"}, {"mountPath": "/etc/tower/settings.py", "name": "settings", "readOnly": true, "subPath": "settings.py"}, {"mountPath": "/etc/nginx/nginx.conf", "name": "nginx-conf", "readOnly": true, "subPath": "nginx.conf"}]}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "terminationGracePeriodSeconds": 30, "volumes": [{"name": "secret-key", "secret": {"defaultMode": 420, "items": [{"key": "secret_key", "path": "SECRET_KEY"}], "secretName": "awx-tower-secret"}}, {"configMap": {"defaultMode": 420, "items": [{"key": "environment", "path": "environment.sh"}], "name": "awx-tower-configmap"}, "name": "environment"}, {"configMap": {"defaultMode": 420, "items": [{"key": "settings", "path": "settings.py"}], "name": "awx-tower-configmap"}, "name": "settings"}, {"configMap": {"defaultMode": 420, "items": [{"key": "nginx_conf", "path": "nginx.conf"}], "name": "awx-tower-configmap"}, "name": "nginx-conf"}]}}}, "status": {}}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "v1", "kind": "Service", "metadata": {"creationTimestamp": "2020-03-19T08:26:37Z", "labels": {"app": "tower"}, "name": "awx-service", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5729", "selfLink": "/api/v1/namespaces/awx/services/awx-service", "uid": "90f806f0-93e5-45fd-9a52-a1b8b90f001a"}, "spec": {"clusterIP": "10.152.183.143", "ports": [{"port": 80, "protocol": "TCP", "targetPort": 8052}], "selector": {"app": "tower"}, "sessionAffinity": "None", "type": "ClusterIP"}, "status": {"loadBalancer": {}}}, "warnings": []}, {"changed": true, "method": "create", "result": {"apichanged: [localhost] => (item=tower_web.yaml.j2) => {"ansible_loop_var": "item", "changed": true, "item": "tower_web.yaml.j2", "result": {"results": [{"changed": true, "method": "create", "result": {"apiVersion": "v1", "data": {"admin_password": "cHdBbjgyIXNp", "secret_key": "YXd4c2VjcmV0"}, "kind": "Secret", "metadata": {"creationTimestamp": "2020-03-19T08:26:37Z", "name": "awx-tower-secret", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5722", "selfLink": "/api/v1/namespaces/awx/secrets/awx-tower-secret", "uid": "1ad7948a-efe5-482f-bd1b-438347200844"}, "type": "Opaque"}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"creationTimestamp": "2020-03-19T08:26:37Z", "generation": 1, "labels": {"app": "tower"}, "name": "awx-tower-web", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5723", "selfLink": "/apis/apps/v1/namespaces/awx/deployments/awx-tower-web", "uid": "fd0a1235-e2c0-4b16-856b-f95b94083831"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 1, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "tower"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"creationTimestamp": null, "labels": {"app": "tower"}}, "spec": {"containers": [{"image": "ansible/awx_web:9.3.0", "imagePullPolicy": "IfNotPresent", "name": "tower", "ports": [{"containerPort": 8052, "protocol": "TCP"}], "resources": {"requests": {"cpu": "1", "memory": "2Gi"}}, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [{"mountPath": "/etc/tower/SECRET_KEY", "name": "secret-key", "readOnly": true, "subPath": "SECRET_KEY"}, {"mountPath": "/etc/tower/conf.d/environment.sh", "name": "environment", "readOnly": true, "subPath": "environment.sh"}, {"mountPath": "/etc/tower/settings.py", "name": "settings", "readOnly": true, "subPath": "settings.py"}, {"mountPath": "/etc/nginx/nginx.conf", "name": "nginx-conf", "readOnly": true, "subPath": "nginx.conf"}]}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "terminationGracePeriodSeconds": 30, "volumes": [{"name": "secret-key", "secret": {"defaultMode": 420, "items": [{"key": "secret_key", "path": "SECRET_KEY"}], "secretName": "awx-tower-secret"}}, {"configMap": {"defaultMode": 420, "items": [{"key": "environment", "path": "environment.sh"}], "name": "awx-tower-configmap"}, "name": "environment"}, {"configMap": {"defaultMode": 420, "items": [{"key": "settings", "path": "settings.py"}], "name": "awx-tower-configmap"}, "name": "settings"}, {"configMap": {"defaultMode": 420, "items": [{"key": "nginx_conf", "path": "nginx.conf"}], "name": "awx-tower-configmap"}, "name": "nginx-conf"}]}}}, "status": {}}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "v1", "kind": "Service", "metadata": {"creationTimestamp": "2020-03-19T08:26:37Z", "labels": {"app": "tower"}, "name": "awx-service", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5729", "selfLink": "/api/v1/namespaces/awx/services/awx-service", "uid": "90f806f0-93e5-45fd-9a52-a1b8b90f001a"}, "spec": {"clusterIP": "10.152.183.143", "ports": [{"port": 80, "protocol": "TCP", "targetPort": 8052}], "selector": {"app": "tower"}, "sessionAffinity": "None", "type": "ClusterIP"}, "status": {"loadBalancer": {}}}, "warnings": []}, {"changed": true, "method": "create", "result": {"apiVersion": "extensions/v1beta1", "kind": "Ingress", "metadata": {"creationTimestamp": "2020-03-19T08:26:37Z", "generation": 1, "name": "awx-ingress", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5739", "selfLink": "/apis/extensions/v1beta1/namespaces/awx/ingresses/awx-ingress", "uid": "a3b64afc-bd34-4f13-a98b-428eafed2ad5"}, "spec": {"rules": [{"host": "microk8s", "http": {"paths": [{"backend": {"serviceName": "awx-service", "servicePort": 80}, "path": "/"}]}}]}, "status": {"loadBalancer": {}}}, "warnings": []}]}}
changed: [localhost] => (item=tower_task.yaml.j2) => {"ansible_loop_var": "item", "changed": true, "item": "tower_task.yaml.j2", "method": "create", "result": {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"creationTimestamp": "2020-03-19T08:26:38Z", "generation": 1, "labels": {"app": "tower-task"}, "name": "awx-tower-task", "namespace": "awx", "ownerReferences": [{"apiVersion": "tower.ansible.com/v1alpha1", "kind": "Tower", "name": "awx", "uid": "fcc3dc9d-0c52-4523-a428-fa5cfa203864"}], "resourceVersion": "5744", "selfLink": "/apis/apps/v1/namespaces/awx/deployments/awx-tower-task", "uid": "671ee6d5-24f4-42fa-928e-e93281581bbf"}, "spec": {"progressDeadlineSeconds": 600, "replicas": 1, "revisionHistoryLimit": 10, "selector": {"matchLabels": {"app": "tower-task"}}, "strategy": {"rollingUpdate": {"maxSurge": "25%", "maxUnavailable": "25%"}, "type": "RollingUpdate"}, "template": {"metadata": {"creationTimestamp": null, "labels": {"app": "tower-task"}}, "spec": {"containers": [{"command": ["/usr/bin/launch_awx_task.sh"], "envFrom": [{"configMapRef": {"name": "awx-tower-configmap"}}, {"secretRef": {"name": "awx-tower-secret"}}], "image": "ansible/awx_task:9.3.0", "imagePullPolicy": "IfNotPresent", "name": "tower-task", "resources": {"requests": {"cpu": "500m", "memory": "1Gi"}}, "securityContext": {"privileged": true}, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [{"mountPath": "/etc/tower/SECRET_KEY", "name": "secret-key", "readOnly": true, "subPath": "SECRET_KEY"}, {"mountPath": "/etc/tower/conf.d/environment.sh", "name": "environment", "readOnly": true, "subPath": "environment.sh"}, {"mountPath": "/etc/tower/settings.py", "name": "settings", "readOnly": true, "subPath": "settings.py"}]}], "dnsPolicy": "ClusterFirst", "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "terminationGracePeriodSeconds": 30, "volumes": [{"name": "secret-key", "secret": {"defaultMode": 420, "items": [{"key": "secret_key", "path": "SECRET_KEY"}], "secretName": "awx-tower-secret"}}, {"configMap": {"defaultMode": 420, "items": [{"key": "environment", "path": "environment.sh"}], "name": "awx-tower-configmap"}, "name": "environment"}, {"configMap": {"defaultMode": 420, "items": [{"key": "settings", "path": "settings.py"}], "name": "awx-tower-configmap"}, "name": "settings"}]}}}, "status": {}}}
TASK [tower : Get the Tower pod information.] **********************************
task path: /opt/ansible/roles/tower/tasks/main.yml:14
ok: [localhost] => {"attempts": 1, "changed": false, "resources": [{"apiVersion": "v1", "kind": "Pod", "metadata": {"creationTimestamp": "2020-03-19T08:26:37Z", "generateName": "awx-tower-web-c98cd6555-", "labels": {"app": "tower", "pod-template-hash": "c98cd6555"}, "name": "awx-tower-web-c98cd6555-jtzfg", "namespace": "awx", "ownerReferences": [{"apiVersion": "apps/v1", "blockOwnerDeletion": true, "controller": true, "kind": "ReplicaSet", "name": "awx-tower-web-c98cd6555", "uid": "77238ee6-e1f4-4610-a65c-701359cc8efd"}], "resourceVersion": "5765", "selfLink": "/api/v1/namespaces/awx/pods/awx-tower-web-c98cd6555-jtzfg", "uid": "77dfe879-08c6-4643-981b-2c456698e89a"}, "spec": {"containers": [{"image": "ansible/awx_web:9.3.0", "imagePullPolicy": "IfNotPresent", "name": "tower", "ports": [{"containerPort": 8052, "protocol": "TCP"}], "resources": {"requests": {"cpu": "1", "memory": "2Gi"}}, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [{"mountPath": "/etc/tower/SECRET_KEY", "name": "secret-key", "readOnly": true, "subPath": "SECRET_KEY"}, {"mountPath": "/etc/tower/conf.d/environment.sh", "name": "environment", "readOnly": true, "subPath": "environment.sh"}, {"mountPath": "/etc/tower/settings.py", "name": "settings", "readOnly": true, "subPath": "settings.py"}, {"mountPath": "/etc/nginx/nginx.conf", "name": "nginx-conf", "readOnly": true, "subPath": "nginx.conf"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "default-token-sqw2l", "readOnly": true}]}], "dnsPolicy": "ClusterFirst", "enableServiceLinks": true, "nodeName": "microk8s", "priority": 0, "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": {}, "serviceAccount": "default", "serviceAccountName": "default", "terminationGracePeriodSeconds": 30, "tolerations": [{"effect": "NoExecute", "key": "node.kubernetes.io/not-ready", "operator": "Exists", "tolerationSeconds": 300}, {"effect": "NoExecute", "key": "node.kubernetes.io/unreachable", "operator": "Exists", "tolerationSeconds": 300}], "volumes": [{"name": "secret-key", "secret": {"defaultMode": 420, "items": [{"key": "secret_key", "path": "SECRET_KEY"}], "secretName": "awx-tower-secret"}}, {"configMap": {"defaultMode": 420, "items": [{"key": "environment", "path": "environment.sh"}], "name": "awx-tower-configmap"}, "name": "environment"}, {"configMap": {"defaultMode": 420, "items": [{"key": "settings", "path": "settings.py"}], "name": "awx-tower-configmap"}, "name": "settings"}, {"configMap": {"defaultMode": 420, "items": [{"key": "nginx_conf", "path": "nginx.conf"}], "name": "awx-tower-configmap"}, "name": "nginx-conf"}, {"name": "default-token-sqw2l", "secret": {"defaultMode": 420, "secretName": "default-token-sqw2l"}}]}, "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2020-03-19T08:26:37Z", "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2020-03-19T08:26:39Z", "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2020-03-19T08:26:39Z", "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2020-03-19T08:26:37Z", "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"containerID": "containerd://c32c89a8657d58ba5529ef0d96679094f0bebbbd61bd4dbd354afdd2e46c160a", "image": "docker.io/ansible/awx_web:9.3.0", "imageID": "docker.io/ansible/awx_web@sha256:e3716cce276a9774650a4fbbb5d80c98fa734db633e8ae4ea661d178c23b89df", "lastState": {}, "name": "tower", "ready": true, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2020-03-19T08:26:39Z"}}}], "hostIP": "192.168.1.119", "phase": "Running", "podIP": "10.1.9.26", "podIPs": [{"ip": "10.1.9.26"}], "qosClass": "Burstable", "startTime": "2020-03-19T08:26:37Z"}}]}
TASK [tower : Set the tower pod name as a variable.] ***************************
TASK [tower : Set the tower pod name as a variable.] ***************************
task path: /opt/ansible/roles/tower/tasks/main.yml:25
ok: [localhost] => {"ansible_facts": {"tower_pod_name": "awx-tower-web-c98cd6555-jtzfg"}, "changed": false}
TASK [tower : Verify tower_pod_name is populated.] *****************************
task path: /opt/ansible/roles/tower/tasks/main.yml:29
ok: [localhost] => {
"changed": false,
"msg": "All assertions passed"
}
TASK [tower : Check if database is populated (auth_user table exists).] ********
task path: /opt/ansible/roles/tower/tasks/main.yml:34
skipping: [localhost] => {"changed": false, "skip_reason": "Conditional result was False"}
TASK [tower : Migrate the database if the K8s resources were updated.] *********
task path: /opt/ansible/roles/tower/tasks/main.yml:46
r/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 323, in run_from_argv\n self.execute(*args, **cmd_options)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 364, in execute\n output = self.handle(*args, **options)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 83, in wrapped\n res = handle_func(*args, **kwargs)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/commands/migrate.py\", line 87, in handle\n executor = MigrationExecutor(connection, self.migration_progress_callback)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/executor.py\", line 18, in __init__\n self.loader = MigrationLoader(self.connection)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 49, in __init__\n self.build_graph()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 212, in build_graph\n self.applied_migrations = recorder.applied_migrations()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/recorder.py\", line 73, in applied_migrations\n if self.has_table():\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/recorder.py\", line 56, in has_table\n return self.Migration._meta.db_table in self.connection.introspection.table_names(self.connection.cursor())\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 256, in cursor\n return self._cursor()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 233, in _cursor\n self.ensure_connection()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection\n self.connect()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/utils.py\", line 89, in __exit__\n raise dj_exc_value.with_traceback(traceback) from exc_value\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection\n self.connect()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 195, in connect\n self.connection = self.get_new_connection(conn_params)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/postgresql/base.py\", line 178, in get_new_connection\n connection = Database.connect(**conn_params)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/psycopg2/__init__.py\", line 126, in connect\n conn = _connect(dsn, connection_factory=connection_factory, **kwasync)\ndjango.db.utils.OperationalError: could not connect to server: Connection timed out\n\tIs the server running on host \"awx-postgres.awx.svc.cluster.local\" (91.201.60.73) and accepting\n\tTCP/IP connections on port 5432?\n\ncommand terminated with exit code 1", "stderr_lines": ["Traceback (most recent call last):", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection", " self.connect()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 195, in connect", " self.connection = self.get_new_connection(conn_params)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/postgresql/base.py\", line 178, in get_new_connection", " connection = Database.connect(**conn_params)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/psycopg2/__init__.py\", line 126, in connect", " conn = _connect(dsn, connection_factory=connection_factory, **kwasync)", "psycopg2.OperationalError: could not connect to server: Connection timed out", "\tIs the server running on host \"awx-postgres.awx.svc.cluster.local\" (91.201.60.73) and accepting", "\tTCP/IP connections on port 5432?", "", "", "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", " File \"/usr/bin/awx-manage\", line 8, in <module>", " sys.exit(manage())", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/awx/__init__.py\", line 152, in manage", " execute_from_command_line(sys.argv)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/__init__.py\", line 381, in execute_from_command_line", " utility.execute()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/__init__.py\", line 375, in execute", " self.fetch_command(subcommand).run_from_argv(self.argv)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 323, in run_from_argv", " self.execute(*args, **cmd_options)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 364, in execute", " output = self.handle(*args, **options)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 83, in wrapped", " res = handle_func(*args, **kwargs)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/commands/migrate.py\", line 87, in handle", " executor = MigrationExecutor(connection, self.migration_progress_callback)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/executor.py\", line 18, in __init__", " self.loader = MigrationLoader(self.connection)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 49, in __init__", " self.build_graph()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 212, in build_graph", " self.applied_migrations = recorder.applied_migrations()", " File \"/var/lib/awx/venv/awx/lib/python3fatal: [localhost]: FAILED! => {"changed": true, "cmd": "kubectl exec -n awx awx-tower-web-c98cd6555-jtzfg -- bash -c \"awx-manage migrate --noinput\"", "delta": "0:19:47.851857", "end": "2020-03-19 08:46:30.636223", "msg": "non-zero return code", "rc": 1, "start": "2020-03-19 08:26:42.784366", "stderr": "Traceback (most recent call last):\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection\n self.connect()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 195, in connect\n self.connection = self.get_new_connection(conn_params)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/postgresql/base.py\", line 178, in get_new_connection\n connection = Database.connect(**conn_params)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/psycopg2/__init__.py\", line 126, in connect\n conn = _connect(dsn, connection_factory=connection_factory, **kwasync)\npsycopg2.OperationalError: could not connect to server: Connection timed out\n\tIs the server running on host \"awx-postgres.awx.svc.cluster.local\" (91.201.60.73) and accepting\n\tTCP/IP connections on port 5432?\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/usr/bin/awx-manage\", line 8, in <module>\n sys.exit(manage())\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/awx/__init__.py\", line 152, in manage\n execute_from_command_line(sys.argv)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/__init__.py\", line 381, in execute_from_command_line\n utility.execute()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/__init__.py\", line 375, in execute\n self.fetch_command(subcommand).run_from_argv(self.argv)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 323, in run_from_argv\n self.execute(*args, **cmd_options)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 364, in execute\n output = self.handle(*args, **options)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 83, in wrapped\n res = handle_func(*args, **kwargs)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/commands/migrate.py\", line 87, in handle\n executor = MigrationExecutor(connection, self.migration_progress_callback)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/executor.py\", line 18, in __init__\n self.loader = MigrationLoader(self.connection)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 49, in __init__\n self.build_graph()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 212, in build_graph\n self.applied_migrations = recorder.applied_migrations()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/recorder.py\", line 73, in applied_migrations\n if self.has_table():\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/recorder.py\", line 56, in has_table\n return self.Migration._meta.db_table in self.connection.introspection.table_names(self.connection.cursor())\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 256, in cursor\n return self._cursor()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 233, in _cursor\n self.ensure_connection()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection\n self.connect()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/utils.py\", line 89, in __exit__\n raise dj_exc_value.with_traceback(traceback) from exc_value\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection\n self.connect()\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 195, in connect\n self.connection = self.get_new_connection(conn_params)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/postgresql/base.py\", line 178, in get_new_connection\n connection = Database.connect(**conn_params)\n File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/psycopg2/__init__.py\", line 126, in connect\n conn = _connect(dsn, connection_factory=connection_factory, **kwasync)\ndjango.db.utils.OperationalError: could not connect to server: Connection timed out\n\tIs the server running on host \"awx-postgres.awx.svc.cluster.local\" (91.201.60.73) and accepting\n\tTCP/IP connections on port 5432?\n\ncommand terminated with exit code 1", "stderr_lines": ["Traceback (most recent call last):", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection", " self.connect()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 195, in connect", " self.connection = self.get_new_connection(conn_params)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/postgresql/base.py\", line 178, in get_new_connection", " connection = Database.connect(**conn_params)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/psycopg2/__init__.py\", line 126, in connect", " conn = _connect(dsn, connection_factory=connection_factory, **kwasync)", "psycopg2.OperationalError: could not connect to server: Connection timed out", "\tIs the server running on host \"awx-postgres.awx.svc.cluster.local\" (91.201.60.73) and accepting", "\tTCP/IP connections on port 5432?", "", "", "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", " File \"/usr/bin/awx-manage\", line 8, in <module>", " sys.exit(manage())", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/awx/__init__.py\", line 152, in manage", " execute_from_command_line(sys.argv)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/__init__.py\", line 381, in execute_from_command_line", " utility.execute()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/__init__.py\", line 375, in execute", " self.fetch_command(subcommand).run_from_argv(self.argv)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 323, in run_from_argv", " self.execute(*args, **cmd_options)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 364, in execute", " output = self.handle(*args, **options)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/base.py\", line 83, in wrapped", " res = handle_func(*args, **kwargs)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/core/management/commands/migrate.py\", line 87, in handle", " executor = MigrationExecutor(connection, self.migration_progress_callback)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/executor.py\", line 18, in __init__", " self.loader = MigrationLoader(self.connection)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 49, in __init__", " self.build_graph()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/loader.py\", line 212, in build_graph", " self.applied_migrations = recorder.applied_migrations()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/recorder.py\", line 73, in applied_migrations", " if self.has_table():", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/migrations/recorder.py\", line 56, in has_table", " return self.Migration._meta.db_table in self.connection.introspection.table_names(self.connection.cursor())", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 256, in cursor", " return self._cursor()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 233, in _cursor", " self.ensure_connection()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection", " self.connect()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/utils.py\", line 89, in __exit__", " raise dj_exc_value.with_traceback(traceback) from exc_value", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 217, in ensure_connection", " self.connect()", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/base/base.py\", line 195, in connect", " self.connection = self.get_new_connection(conn_params)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/django/db/backends/postgresql/base.py\", line 178, in get_new_connection", " connection = Database.connect(**conn_params)", " File \"/var/lib/awx/venv/awx/lib/python3.6/site-packages/psycopg2/__init__.py\", line 126, in connect", " conn = _connect(dsn, connection_factory=connection_factory, **kwasync)", "django.db.utils.OperationalError: could not connect to server: Connection timed out", "\tIs the server running on host \"awx-postgres.awx.svc.cluster.local\" (91.201.60.73) and accepting", "\tTCP/IP connections on port 5432?", "", "command terminated with exit code 1"], "stdout": "", "stdout_lines": []}
PLAY RECAP *********************************************************************
localhost : ok=4 changed=1 unreachable=0 failed=1 skipped=1 rescued=0 ignored=0