This job view page is being replaced by Spyglass soon. Check out the new job view.
PRkkasravi: Apply refactor
ResultFAILED
Tests 2 failed / 59 succeeded
Started2019-08-21 23:44
Elapsed25m4s
Revision
Refs master:840e24b5
3957:4ca04f82
kubeflow-presubmit-dashboard-release-3957-4ca04f8-6721-f3a8-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-dashboard-release-3957-4ca04f8-6721-f3a8?tab=workflow
kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2?tab=workflow
kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8?tab=workflow
kubeflow-presubmit-tf-serving-3957-4ca04f8-6721-7a8b-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-tf-serving-3957-4ca04f8-6721-7a8b?tab=workflow
kubeflow-presubmit-unittests-3957-4ca04f8-6721-bf15-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-unittests-3957-4ca04f8-6721-bf15?tab=workflow
kubeflow-presubmit-dashboard-release-3957-4ca04f8-6721-f3a8-phaseSucceeded
kubeflow-presubmit-dashboard-release-3957-4ca04f8-6721-f3a8-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-dashboard-release-3957-4ca04f8-6721-f3a8?tab=workflow
kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2-phaseFailed
kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2?tab=workflow
kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8-phaseFailed
kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8?tab=workflow
kubeflow-presubmit-tf-serving-3957-4ca04f8-6721-7a8b-phaseSucceeded
kubeflow-presubmit-tf-serving-3957-4ca04f8-6721-7a8b-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-tf-serving-3957-4ca04f8-6721-7a8b?tab=workflow
kubeflow-presubmit-unittests-3957-4ca04f8-6721-bf15-phaseSucceeded
kubeflow-presubmit-unittests-3957-4ca04f8-6721-bf15-uihttp://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/kubeflow-presubmit-unittests-3957-4ca04f8-6721-bf15?tab=workflow
repos{u'kubeflow/kubeflow': u'master:840e24b5e86654d8e4a3596a3c91ffb2d198bca1,3957:4ca04f82d48c1cce65a8ad29fc973bed8b27d4a9'}

Test Failures


test_kfctl_go_deploy_basic-auth-istio test_build_kfctl_go 13m7s

go run hack/e2e.go -v --test --test_args='--ginkgo.focus=test\_kfctl\_go\_deploy\_basic\-auth\-istio\stest\_build\_kfctl\_go$'
app_path = '/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2/kfctl-31b2'
project = 'kubeflow-ci-deployment', use_basic_auth = True, use_istio = True
config_path = '/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2/src/kubeflow/kubeflow/bootstrap/config/kfctl_gcp_basic_auth.yaml'

    def test_build_kfctl_go(app_path, project, use_basic_auth, use_istio, config_path):
      """Test building and deploying Kubeflow.
    
      Args:
        app_path: The path to the Kubeflow app.
        project: The GCP project to use.
      """
      if not app_path:
        logging.info("--app_path not specified")
        stamp = datetime.datetime.now().strftime("%H%M")
        parent_dir = tempfile.gettempdir()
        app_path = os.path.join(
            parent_dir, "kfctl-{0}-{1}".format(stamp,
                                               uuid.uuid4().hex[0:4]))
      else:
        parent_dir = os.path.dirname(app_path)
    
      logging.info("Using app path %s", app_path)
      this_dir = os.path.dirname(__file__)
      root = os.path.abspath(os.path.join(this_dir, "..", ".."))
      build_dir = os.path.join(root, "bootstrap")
      zone = 'us-central1-a'
    
      # Need to activate account for scopes.
      if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"):
        util.run([
            "gcloud", "auth", "activate-service-account",
            "--key-file=" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
        ])
    
      # We need to use retry builds because when building in the test cluster
      # we see intermittent failures pulling dependencies
      run_with_retries(["make", "build-kfctl"], cwd=build_dir)
      kfctl_path = os.path.join(build_dir, "bin", "kfctl")
    
      # Set ENV for basic auth username/password.
      init_args = []
      if use_basic_auth:
        os.environ["KUBEFLOW_USERNAME"] = "kf-test-user"
        os.environ["KUBEFLOW_PASSWORD"] = str(uuid.uuid4().hex)
        init_args = ["--use_basic_auth"]
      else:
        # Owned by project kubeflow-ci-deployment.
        os.environ["CLIENT_SECRET"] = "CJ4qVPLTi0j0GJMkONj7Quwt"
        os.environ["CLIENT_ID"] = (
          "29647740582-7meo6c7a9a76jvg54j0g2lv8lrsb4l8g"
          ".apps.googleusercontent.com")
    
      if use_istio:
        init_args.append("--use_istio")
      else:
        init_args.append("--use_istio=false")
    
      version = "master"
      if os.getenv("REPO_NAME") != "manifests":
        if os.getenv("PULL_NUMBER"):
          version = "pull/{0}".format(os.getenv("PULL_NUMBER"))
      pull_manifests = "@master"
      if os.getenv("REPO_NAME") == "manifests":
        if os.getenv("PULL_PULL_SHA"):
          pull_manifests = "@" + os.getenv("PULL_PULL_SHA")
    
      # We need to specify a valid email because
      #  1. We need to create appropriate RBAC rules to allow the current user
      #     to create the required K8s resources.
      #  2. Setting the IAM policy will fail if the email is invalid.
      email = util.run(["gcloud", "config", "get-value", "account"])
    
      if not email:
        raise ValueError("Could not determine GCP account being used.")
    
      # username and password are passed as env vars and won't appear in the logs
      # TODO(https://github.com/kubeflow/kubeflow/issues/2831): Once kfctl
      # supports loading version from a URI we should use that so that we
      # pull the configs from the repo we checked out.
      #
      # We don't run with retries because if kfctl init exits with an error
      # but creates app.yaml then rerunning init will fail because app.yaml
      # already exists. So retrying ends up masking the original error message
      with open(config_path, 'r') as f:
        config_spec = yaml.load(f)
      config_spec["spec"]["project"] = project
      config_spec["spec"]["email"] = email
      config_spec["spec"] = filterSpartakus(config_spec["spec"])
    
      logging.info(str(config_spec))
      with open(os.path.join(parent_dir, "tmp.yaml"), "w") as f:
        yaml.dump(config_spec, f)
      util.run([
          kfctl_path, "init", app_path, "-V",
          "--config=" + os.path.join(parent_dir, "tmp.yaml")], cwd=parent_dir)
      util.run(["cat", "app.yaml"], cwd=app_path)
    
      run_with_retries([
          kfctl_path, "generate", "-V", "all", "--email=" + email, "--zone=" + zone
      ],
                       cwd=app_path)
    
      # We need to use retries because if we don't we see random failures
      # where kfctl just appears to die.
      #
      # Do not run with retries since it masks errors
>     util.run([kfctl_path, "apply", "-V", "all"], cwd=app_path)

kfctl_go_test.py:135: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

command = ['/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2/src/kubeflow/kubeflow/bootstrap/bin/kfctl', 'apply', '-V', 'all']
cwd = '/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2/kfctl-31b2'
env = environ({'ARGO_UI_SERVICE_HOST': '10.39.246.223', 'KUBEFLOW_USERNAME': 'kf-test-user', 'USER': 'root', 'REPO_OWNER': '...1', 'ARGO_UI_PORT_80_TCP_PROTO': 'tcp', 'HOSTNAME': 'kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2-2'})
polling_interval = datetime.timedelta(0, 1)

    def run(command,
            cwd=None,
            env=None,
            polling_interval=datetime.timedelta(seconds=1)):
      """Run a subprocess.
    
      Any subprocess output is emitted through the logging modules.
    
      Returns:
        output: A string containing the output.
      """
      logging.info("Running: %s \ncwd=%s", " ".join(command), cwd)
    
      if not env:
        env = os.environ
      else:
        keys = sorted(env.keys())
    
        lines = []
        for k in keys:
          lines.append("{0}={1}".format(k, env[k]))
        logging.info("Running: Environment:\n%s", "\n".join(lines))
    
      process = subprocess.Popen(
        command, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    
      logging.info("Subprocess output:\n")
      output = []
      while process.poll() is None:
        process.stdout.flush()
        for line in iter(process.stdout.readline, b''):
          if six.PY2:
            line = line.strip()
          else:
            line = line.decode().strip()
    
          output.append(line)
          logging.info(line)
    
        time.sleep(polling_interval.total_seconds())
    
      process.stdout.flush()
      for line in iter(process.stdout.readline, b''):
        if six.PY2:
          line = line.strip()
        else:
          line = line.decode().strip()
        output.append(line)
        logging.info(line)
    
      if process.returncode != 0:
        raise subprocess.CalledProcessError(
          process.returncode, "cmd: {0} exited with code {1}".format(
>           " ".join(command), process.returncode), "\n".join(output))
E       subprocess.CalledProcessError: Command 'cmd: /mnt/test-data-volume/kubeflow-presubmit-kfctl-go-basic-auth-3957-4ca04f8-6721-31b2/src/kubeflow/kubeflow/bootstrap/bin/kfctl apply -V all exited with code 1' returned non-zero exit status 1

../../../testing/py/kubeflow/testing/util.py:87: CalledProcessError
				from junit_kfctl-build-testbasic-auth-istio.xml

Filter through log files | View test history on testgrid


test_kfctl_go_deploy_iap-istio test_build_kfctl_go 14m35s

go run hack/e2e.go -v --test --test_args='--ginkgo.focus=test\_kfctl\_go\_deploy\_iap\-istio\stest\_build\_kfctl\_go$'
app_path = '/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8/kfctl-09a8'
project = 'kubeflow-ci-deployment', use_basic_auth = False, use_istio = True
config_path = '/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8/src/kubeflow/kubeflow/bootstrap/config/kfctl_gcp_iap.yaml'

    def test_build_kfctl_go(app_path, project, use_basic_auth, use_istio, config_path):
      """Test building and deploying Kubeflow.
    
      Args:
        app_path: The path to the Kubeflow app.
        project: The GCP project to use.
      """
      if not app_path:
        logging.info("--app_path not specified")
        stamp = datetime.datetime.now().strftime("%H%M")
        parent_dir = tempfile.gettempdir()
        app_path = os.path.join(
            parent_dir, "kfctl-{0}-{1}".format(stamp,
                                               uuid.uuid4().hex[0:4]))
      else:
        parent_dir = os.path.dirname(app_path)
    
      logging.info("Using app path %s", app_path)
      this_dir = os.path.dirname(__file__)
      root = os.path.abspath(os.path.join(this_dir, "..", ".."))
      build_dir = os.path.join(root, "bootstrap")
      zone = 'us-central1-a'
    
      # Need to activate account for scopes.
      if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"):
        util.run([
            "gcloud", "auth", "activate-service-account",
            "--key-file=" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
        ])
    
      # We need to use retry builds because when building in the test cluster
      # we see intermittent failures pulling dependencies
      run_with_retries(["make", "build-kfctl"], cwd=build_dir)
      kfctl_path = os.path.join(build_dir, "bin", "kfctl")
    
      # Set ENV for basic auth username/password.
      init_args = []
      if use_basic_auth:
        os.environ["KUBEFLOW_USERNAME"] = "kf-test-user"
        os.environ["KUBEFLOW_PASSWORD"] = str(uuid.uuid4().hex)
        init_args = ["--use_basic_auth"]
      else:
        # Owned by project kubeflow-ci-deployment.
        os.environ["CLIENT_SECRET"] = "CJ4qVPLTi0j0GJMkONj7Quwt"
        os.environ["CLIENT_ID"] = (
          "29647740582-7meo6c7a9a76jvg54j0g2lv8lrsb4l8g"
          ".apps.googleusercontent.com")
    
      if use_istio:
        init_args.append("--use_istio")
      else:
        init_args.append("--use_istio=false")
    
      version = "master"
      if os.getenv("REPO_NAME") != "manifests":
        if os.getenv("PULL_NUMBER"):
          version = "pull/{0}".format(os.getenv("PULL_NUMBER"))
      pull_manifests = "@master"
      if os.getenv("REPO_NAME") == "manifests":
        if os.getenv("PULL_PULL_SHA"):
          pull_manifests = "@" + os.getenv("PULL_PULL_SHA")
    
      # We need to specify a valid email because
      #  1. We need to create appropriate RBAC rules to allow the current user
      #     to create the required K8s resources.
      #  2. Setting the IAM policy will fail if the email is invalid.
      email = util.run(["gcloud", "config", "get-value", "account"])
    
      if not email:
        raise ValueError("Could not determine GCP account being used.")
    
      # username and password are passed as env vars and won't appear in the logs
      # TODO(https://github.com/kubeflow/kubeflow/issues/2831): Once kfctl
      # supports loading version from a URI we should use that so that we
      # pull the configs from the repo we checked out.
      #
      # We don't run with retries because if kfctl init exits with an error
      # but creates app.yaml then rerunning init will fail because app.yaml
      # already exists. So retrying ends up masking the original error message
      with open(config_path, 'r') as f:
        config_spec = yaml.load(f)
      config_spec["spec"]["project"] = project
      config_spec["spec"]["email"] = email
      config_spec["spec"] = filterSpartakus(config_spec["spec"])
    
      logging.info(str(config_spec))
      with open(os.path.join(parent_dir, "tmp.yaml"), "w") as f:
        yaml.dump(config_spec, f)
      util.run([
          kfctl_path, "init", app_path, "-V",
          "--config=" + os.path.join(parent_dir, "tmp.yaml")], cwd=parent_dir)
      util.run(["cat", "app.yaml"], cwd=app_path)
    
      run_with_retries([
          kfctl_path, "generate", "-V", "all", "--email=" + email, "--zone=" + zone
      ],
                       cwd=app_path)
    
      # We need to use retries because if we don't we see random failures
      # where kfctl just appears to die.
      #
      # Do not run with retries since it masks errors
>     util.run([kfctl_path, "apply", "-V", "all"], cwd=app_path)

kfctl_go_test.py:135: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

command = ['/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8/src/kubeflow/kubeflow/bootstrap/bin/kfctl', 'apply', '-V', 'all']
cwd = '/mnt/test-data-volume/kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8/kfctl-09a8'
env = environ({'DEBIAN_FRONTEND': 'noninteractive', 'ARGO_UI_SERVICE_PORT': '80', 'PATH': 'root/bin:/root/go/bin:/usr/local/..._80_TCP_ADDR': '10.39.246.223', 'KUBERNETES_PORT_443_TCP_PROTO': 'tcp', 'KUBERNETES_PORT_443_TCP_ADDR': '10.39.240.1'})
polling_interval = datetime.timedelta(0, 1)

    def run(command,
            cwd=None,
            env=None,
            polling_interval=datetime.timedelta(seconds=1)):
      """Run a subprocess.
    
      Any subprocess output is emitted through the logging modules.
    
      Returns:
        output: A string containing the output.
      """
      logging.info("Running: %s \ncwd=%s", " ".join(command), cwd)
    
      if not env:
        env = os.environ
      else:
        keys = sorted(env.keys())
    
        lines = []
        for k in keys:
          lines.append("{0}={1}".format(k, env[k]))
        logging.info("Running: Environment:\n%s", "\n".join(lines))
    
      process = subprocess.Popen(
        command, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    
      logging.info("Subprocess output:\n")
      output = []
      while process.poll() is None:
        process.stdout.flush()
        for line in iter(process.stdout.readline, b''):
          if six.PY2:
            line = line.strip()
          else:
            line = line.decode().strip()
    
          output.append(line)
          logging.info(line)
    
        time.sleep(polling_interval.total_seconds())
    
      process.stdout.flush()
      for line in iter(process.stdout.readline, b''):
        if six.PY2:
          line = line.strip()
        else:
          line = line.decode().strip()
        output.append(line)
        logging.info(line)
    
      if process.returncode != 0:
        raise subprocess.CalledProcessError(
          process.returncode, "cmd: {0} exited with code {1}".format(
>           " ".join(command), process.returncode), "\n".join(output))
E       subprocess.CalledProcessError: Command 'cmd: /mnt/test-data-volume/kubeflow-presubmit-kfctl-go-iap-istio-3957-4ca04f8-6721-09a8/src/kubeflow/kubeflow/bootstrap/bin/kfctl apply -V all exited with code 1' returned non-zero exit status 1

../../../testing/py/kubeflow/testing/util.py:87: CalledProcessError
				from junit_kfctl-build-testiap-istio.xml

Filter through log files | View test history on testgrid


Show 59 Passed Tests

Show 1 Skipped Tests