Waiting for vault to be initialized and unsealed... about terraform-aws-eks-jx

[andrewhoplife]

jenkins-x.yml

buildPack: none
pipelineConfig:
  pipelines:
    pullRequest:
      pipeline:
        agent:
          image: gcr.io/jenkinsxio/builder-go
        stages:
        - name: pr-checks
          options:
            containerOptions:
              name: ""
              resources: {}
              volumeMounts:
              - mountPath: /builder/home/.jx/localSecrets/currentCluster
                name: local-secrets
                readOnly: true
            volumes:
            - name: local-secrets
              secret:
                optional: true
                secretName: local-param-secrets
          steps:
          - args:
            - step
            - verify
            - values
            - --values-file=parameters.yaml
            - --schema-file=parameters.tmpl.schema.json
            command: jx
            dir: /workspace/source/env
            name: verify-parameters
          - args:
            - build
            command: make
            dir: /workspace/source/env
            name: lint-env-helm
    release:
      pipeline:
        agent:
          image: gcr.io/jenkinsxio/builder-go
        environment:
        - name: DEPLOY_NAMESPACE
          value: jx
        stages:
        - name: release
          options:
            containerOptions:
              name: ""
              resources: {}
              volumeMounts:
              - mountPath: /builder/home/.jx/localSecrets/currentCluster
                name: local-secrets
                readOnly: true
            volumes:
            - name: local-secrets
              secret:
                optional: true
                secretName: local-param-secrets
          steps:
          - args:
            - step
            - git
            - validate
            command: jx
            dir: /workspace/source/env
            name: validate-git
          - args:
            - step
            - verify
            - preinstall
            - --provider-values-dir="kubeProviders"
            command: jx
            dir: /workspace/source
            name: verify-preinstall
          - args:
            - upgrade
            - crd
            command: jx
            name: install-jx-crds
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --no-vault
            - --name
            - velero
            command: jx
            dir: /workspace/source/systems/velero
            env:
            - name: DEPLOY_NAMESPACE
              value: velero
            name: install-velero
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --no-vault
            - --name
            - velero-backups
            command: jx
            dir: /workspace/source/systems/velero-backups
            env:
            - name: DEPLOY_NAMESPACE
              value: velero
            name: install-velero-backups
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --no-vault
            - --name
            - jxing
            command: jx
            dir: /workspace/source/systems/jxing
            env:
            - name: DEPLOY_NAMESPACE
              value: kube-system
            name: install-nginx-controller
          - args:
            - step
            - create
            - install
            - values
            - -b
            command: jx
            dir: /workspace/source/env
            name: create-install-values
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --no-vault
            - --name
            - exdns
            command: jx
            dir: /workspace/source/systems/external-dns
            name: install-external-dns
          - args:
            - apply
            - --wait
            - --validate=false
            - -f
            - https://raw.githubusercontent.com/jetstack/cert-manager/release-0.11/deploy/manifests/00-crds.yaml
            command: kubectl
            dir: /workspace/source
            env:
            - name: DEPLOY_NAMESPACE
              value: cert-manager
            name: install-cert-manager-crds
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --no-vault
            - --name
            - cm
            command: jx
            dir: /workspace/source/systems/cm
            env:
            - name: DEPLOY_NAMESPACE
              value: cert-manager
            name: install-cert-manager
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --no-vault
            - --name
            - acme
            command: jx
            dir: /workspace/source/systems/acme
            name: install-acme-issuer-and-certificate
          - args:
            - step
            - boot
            - vault
            - --provider-values-dir
            - ../../kubeProviders
            command: jx
            dir: /workspace/source/systems/vault
            name: install-vault
          - args:
            - step
            - create
            - values
            - --name
            - parameters
            command: jx
            dir: /workspace/source/env
            name: create-helm-values
          - args:
            - step
            - create
            - templated
            - --parameters-file=../../env/parameters.yaml
            - --requirements-dir=../../
            - --template-file=jx-auth-configmap.tmpl.yaml
            - --config-file=templates/jx-auth-configmap.yaml
            command: jx
            dir: /workspace/source/systems/jx-auth
            name: create-jx-auth-config
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --no-vault
            - --name
            - jx-auth
            command: jx
            dir: /workspace/source/systems/jx-auth
            name: install-jx-auth-config
          - args:
            - step
            - helm
            - apply
            - --boot
            - --remote
            - --name
            - jenkins-x
            - --provider-values-dir
            - ../kubeProviders
            command: jx
            dir: /workspace/source/env
            name: install-jenkins-x
          - args:
            - step
            - verify
            - env
            command: jx
            dir: /workspace/source
            name: verify-jenkins-x-environment
          - args:
            - step
            - helm
            - apply
            - --boot
            - --name
            - repos
            command: jx
            dir: /workspace/source/repositories
            name: install-repositories
          - args:
            - step
            - scheduler
            - config
            - apply
            - --direct=true
            command: jx
            dir: /workspace/source/prowConfig
            name: install-pipelines
          - args:
            - update
            - webhooks
            - --verbose
            - --warn-on-fail
            command: jx
            dir: /workspace/source/repositories
            name: update-webhooks
          - args:
            - step
            - verify
            - install
            - --pod-wait-time
            - 30m
            command: jx
            dir: /workspace/source/env
            name: verify-installation

[ankitm123]

Do you have the core-dns pods running (most probably a dns resolution issue, kubernetes cannot discover those pods by the dns names)? Any logs from the vault/core-dns pods?

[andrewhoplife]

The only vault/core-dns pods I have is named below. What I think is happening is that a route53 record is not getting created as needed. All that gets created in my subdomain is a SOA and NS record. The documentation does not mention what it is supposed to be but I expect a wildcard for vault-jx that is an A or cname to point to the eks cluster. I tried creating one manually and it does not work.

kubectl describe pod exdns-external-dns-7c686cd8d6-bx7ck                                                         
Name:           exdns-external-dns-7c686cd8d6-bx7ck
Namespace:      jx
Priority:       0
Node:           ip-10-0-1-154.ec2.internal/10.0.1.154
Start Time:     Thu, 04 Jun 2020 15:21:16 -0500
Labels:         app.kubernetes.io/instance=exdns
                app.kubernetes.io/managed-by=Tiller
                app.kubernetes.io/name=external-dns
                helm.sh/chart=external-dns-3.1.1
                pod-template-hash=7c686cd8d6
Annotations:    kubernetes.io/psp: eks.privileged
Status:         Running
IP:             10.0.1.197
IPs:            <none>
Controlled By:  ReplicaSet/exdns-external-dns-7c686cd8d6
Containers:
  external-dns:
    Container ID:  docker://3a4f8fbaeb48124cbabddc28b257ec8bed0cef04a49279c1751d06dec901f3f2
    Image:         docker.io/bitnami/external-dns:0.7.2-debian-10-r0
    Image ID:      docker-pullable://bitnami/external-dns@sha256:47bb59cbe19611d9c474b387a1533ea80d4cb56aa33b8fb3c60c7086dc316f50
    Port:          7979/TCP
    Host Port:     0/TCP
    Args:
      --log-level=info
      --log-format=text
      --policy=upsert-only
      --provider=aws
      --registry=txt
      --interval=1m
      --source=ingress
      --aws-batch-change-size=1000
    State:          Running
      Started:      Thu, 04 Jun 2020 15:21:24 -0500
    Ready:          True
    Restart Count:  0
    Liveness:       http-get http://:http/healthz delay=10s timeout=5s period=10s #success=1 #failure=2
    Readiness:      http-get http://:http/healthz delay=5s timeout=5s period=10s #success=1 #failure=6
    Environment:
      AWS_DEFAULT_REGION:           us-east-1
      AWS_ROLE_ARN:                 arn:aws:iam::535580006495:role/tf-hl-jx-sa-role-external_dns-0WyhSGT3
      AWS_WEB_IDENTITY_TOKEN_FILE:  /var/run/secrets/eks.amazonaws.com/serviceaccount/token
    Mounts:
      /var/run/secrets/eks.amazonaws.com/serviceaccount from aws-iam-token (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from exdns-external-dns-token-zj8dk (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  aws-iam-token:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  86400
  exdns-external-dns-token-zj8dk:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  exdns-external-dns-token-zj8dk
    Optional:    false
QoS Class:       BestEffort
Node-Selectors:  <none>
Tolerations:     node.kubernetes.io/not-ready:NoExecute for 300s
                 node.kubernetes.io/unreachable:NoExecute for 300s
Events:
  Type    Reason     Age    From                                 Message
  ----    ------     ----   ----                                 -------
  Normal  Scheduled  9m12s  default-scheduler                    Successfully assigned jx/exdns-external-dns-7c686cd8d6-bx7ck to ip-10-0-1-154.ec2.internal
  Normal  Pulling    9m10s  kubelet, ip-10-0-1-154.ec2.internal  Pulling image "docker.io/bitnami/external-dns:0.7.2-debian-10-r0"
  Normal  Pulled     9m5s   kubelet, ip-10-0-1-154.ec2.internal  Successfully pulled image "docker.io/bitnami/external-dns:0.7.2-debian-10-r0"
  Normal  Created    9m4s   kubelet, ip-10-0-1-154.ec2.internal  Created container external-dns
  Normal  Started    9m4s   kubelet, ip-10-0-1-154.ec2.internal  Started container external-dns

[joebertj]

https://vault-jx.snipv1/sys/health?drsecondarycode=299&performancestandbycode=299&sealedcode=299&standbycode=299&uninitcode=299: dial tcp: lookup vault-jx.snip on snip:53

I think this is misconfiguration on your side. vault-jx.snipv1 should be vault-jx.snip**/*v1

What you can do is spawn a pod and try to access that URL given that it is not a vaild public DNS.

And you are using https which is not possible to do automatic certificate for since you are using a private local DNS.

Disable DNS for now in your jx-requirements.yaml so you will use nip.io

ingress:
  domain: ""
  ignoreLoadBalancer: true
  externalDNS: false

[andrewhoplife]

I think this is misconfiguration on your side. vault-jx.snipv1 should be vault-jx.snip**/*v1

I am pretty sure that is just a typo in my writeup. Thank you for the disable dns idea will try that today.

[andrewhoplife]

Yeah same result with externaldns set to false. The URL is unavailable on public internet. Pods display no error messages. The only other relevant setting I can think of is ignoreLoadBalancer: true but I don't think that should matter.

STEP: install-vault command: /bin/sh -c jx step boot vault --provider-values-dir ../../kubeProviders in dir: /Users/a/code/jxmanage/jenkins-x-boot-config/systems/vault

? Do you want Jenkins X to create and manage Vault? Yes

Installing vault-operator operator with helm values: [image.repository=banzaicloud/vault-operator image.tag=0.5.3]

Vault operator installed in namespace jx
Applying vault ingress in namespace jx for vault name jx-vault-tf-jx-pumped
ingress.extensions/jx-vault-tf-jx-pumped created
Vault 'jx-vault-tf-jx-pumped' in namespace 'jx' created 

STEP: create-helm-values command: /bin/sh -c jx step create values --name parameters in dir: /Users/a/code/jxmanage/jenkins-x-boot-config/env

defaulting to secret storage scheme vault found from requirements file at /Users/a/code/jxmanage/jenkins-x-boot-config/jx-requirements.yml
defaulting to secret base path to the cluster name tf-jx-pumped-whale found from requirements file at /Users/a/code/jxmanage/jenkins-x-boot-config/jx-requirements.yml
generated schema file /Users/a/code/jxmanage/jenkins-x-boot-config/env/parameters.schema.json from template /Users/a/code/jxmanage/jenkins-x-boot-config/env/parameters.tmpl.schema.json
Waiting for vault to be initialized and unsealed...

error: creating system vault URL client: wait for vault to be initialized and unsealed: reading vault health: Get https://vault-jx.snip/v1/sys/health?drsecondarycode=299&performancestandbycode=299&sealedcode=299&standbycode=299&uninitcode=299: dial tcp: lookup vault-jx.snip on snip:53: server misbehaving
error: failed to interpret pipeline file /Users/a/code/jxmanage/jenkins-x-boot-config/jenkins-x.yml: failed to run '/bin/sh -c jx step create values --name parameters' command in directory '/Users/a/code/jxmanage/jenkins-x-boot-config/env', output: ''

[andrewhoplife]

A dns record is active for vault-jx pointing to the EKS resource. This happens with Vault.AutoCreate set to either true or false. The correct env variable for vault_user_secret and vault_user_id is set.

using version 2.1.62 of jx
CLI packages kubectl, git, helm seem to be setup correctly
NAME               VERSION
jx                 2.1.62
Kubernetes cluster v1.15.11-eks-af3caf
kubectl            v1.13.2
git                2.23.0

? Do you want Jenkins X to create and manage Vault? Yes

Installing vault-operator operator with helm values: [image.repository=banzaicloud/vault-operator image.tag=0.5.3]

Vault operator installed in namespace jx
Applying vault ingress in namespace jx for vault name jx-vault-tf-jx-welcome
ingress.extensions/jx-vault-tf-jx-welcome unchanged
WARNING: Vault.AutoCreate is false but required property secretAccessKey is missing
Some of the required provided values are empty - We will create all resources
Creating vault resources with following values, us-east-1, jenkins-x-vault, vault-data, vault-unseal-tf-jx-welcomed-weevil-20200609175638948300000006
Vault CloudFormation stack created
You can watch progress in the CloudFormation console: https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/stackinfo?stackId=arn:aws:cloudformation:us-east-1:535580006495:stack/jenkins-x-vault-stack9c09e0f/749fe340-aa81-11ea-8ba0-0abd62a7624a
error: unable to create/update Vault: unable to set cloud provider specific Vault configuration: unable to apply cloud provider config: an error occurred while creating the vaultCRD resources: executing the Vault CloudFormation : unable to create vault prerequisite resources: ResourceNotReady: failed waiting for successful resource state
error: failed to interpret pipeline file /Users/a/code/jxmanage/jenkins-x-boot-config/jenkins-x.yml: failed to run '/bin/sh -c jx step boot vault --provider-values-dir ../../kubeProviders' command in directory '/Users/a/code/jxmanage/jenkins-x-boot-config/systems/vault', output: ''

[jenkins-x-bot]

Issues go stale after 90d of inactivity.
Mark the issue as fresh with /remove-lifecycle stale.
Stale issues rot after an additional 30d of inactivity and eventually close.
If this issue is safe to close now please do so with /close.
Provide feedback via https://jenkins-x.io/community.
/lifecycle stale

[jenkins-x-bot]

Stale issues rot after 30d of inactivity.
Mark the issue as fresh with /remove-lifecycle rotten.
Rotten issues close after an additional 30d of inactivity.
If this issue is safe to close now please do so with /close.
Provide feedback via https://jenkins-x.io/community.
/lifecycle rotten

[jenkins-x-bot]

Rotten issues close after 30d of inactivity.
Reopen the issue with /reopen.
Mark the issue as fresh with /remove-lifecycle rotten.
Provide feedback via https://jenkins-x.io/community.
/close

[jenkins-x-bot]

@jenkins-x-bot: Closing this issue.

In response to this:

Rotten issues close after 30d of inactivity.
Reopen the issue with /reopen.
Mark the issue as fresh with /remove-lifecycle rotten.
Provide feedback via https://jenkins-x.io/community.
/close

Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the jenkins-x/lighthouse repository.