Running multiple scenarios at once

k6, Kubernetes

Our test pack is configured dynamically from environment variables. Each scenario can be configured independently with different target VUs, duration or even executor.

Let’s start from a file called main.js. It imports all our scenarios, each as a default function:

export { default as cacheCreateAll } from './runners/cacheCreateAll.js';
export { default as cacheCreateUpdateRemove } from './runners/cacheCreateUpdateRemove.js';
export { default as userSearch } from './runners/userSearch.js';

The main.js file is our entry point to the application. It reads environment variables to configure the scenarios dynamically, sometimes I want to run scenarios in a pack, another time run a single scenario. I used env vars to achieve that flexibility and compatibility with cloud-native approach. I provide list of scenarios in a comma-separated format from an env var.

export function setDefaultValue(param, defaultValue) {
  if (param === null || param === undefined || param === '') {
    return defaultValue;
  } else {
    return param;
  }
}

const scenarios = setDefaultValue(__ENV.PERFORMANCE_SCENARIOS, 'userSearch').split(',');
const targetVirtualUsers = setDefaultValue(__ENV.PERFORMANCE_TARGET_VIRTUAL_USERS, 100);

Then I define the list of scenarios object and a function to configure each scenario:

let dynamicScenarios = {};

function getScenarioConfiguration(exec, target, timeUnit) {
  return {
    exec: exec,

    // https://k6.io/docs/using-k6/scenarios/executors/constant-arrival-rate/
    executor: 'constant-arrival-rate',

    // Our test should last `performanceConstantLoadTime` seconds in total
    duration: performanceConstantLoadTime,

    // It should start `target` iterations per `timeUnit`. Note that iterations starting points
    // will be evenly spread across the `timeUnit` period.
    rate: target,

    // It should start `rate` iterations per second
    timeUnit: timeUnit, // rate of requests spread over timeUnit => evenly spread 60 requests over 60 seconds

    // It should preallocate 100% of VUs before starting the test
    preAllocatedVUs: target,

    // It is allowed to spin up to 140% maximum VUs to sustain the defined
    // constant arrival rate.
    maxVUs: Math.ceil(target * 1.4),

    gracefulStop: '120s'
  };
}

This approach also allows us to run multiple scenarios under one pre-configured name:

if (scenarios.includes('testCreateCaches')) {
  scenarios.push(
    'cacheCreateUpdateRemove',
    'cacheCreateAll'
  );
}

Then based on the scenarios array I populate the dynamicScenarios variable:

if (scenarios.includes('userSearch')) {
  dynamicScenarios.userSearch = getScenarioConfiguration('userSearch', targetVirtualUsers, '250s');
}

or I can configure each scenario with an independent executor:

if (scenarios.includes('userSearch')) {
  dynamicScenarios.userSearch = {
    executor: 'constant-vus',
    vus: targetVirtualUsers,
    duration: performanceConstantLoadTime,
    exec: 'userSearch'
  };
}

And then finally, we put it all inside ak6 configuration:

export let options = {
  maxRedirects: 0, // here I can define global options, thresholds or scenario specific thresholds
  scenarios: dynamicScenarios,
  thresholds: {
    http_req_failed: [{
        threshold: 'rate<0.01',
        abortOnFail: true,
        delayAbortEval: '0s',
    }],
    // 'checks{critical:critical}': [
    //   {
    //     threshold: 'rate>0.99',
    //     abortOnFail: true,
    //     delayAbortEval: '0s'
    //   }
    // ],
    // 'checks{critical:incorrectUserSearchResponse}': [
    //   {
    //     threshold: 'rate>0.99',
    //     abortOnFail: false,
    //     delayAbortEval: '600s'
    //   }
    // ]
  }
};

When I execute k6 run main.js -e PERFORMANCE_SCENARIOS=testCreateCaches the scenarios run in parallel:

running (08m21.1s), 630/1260 VUs, 562002 complete and 0 interrupted iterations
cacheCreateUpdateRemove   [ 50% ] 315 VUs  4m8.1s/8m20s
cacheCreateAll            [ 50% ] 315 VUs  4m8.1s/8m20s

That’s not everything. We can also run each scenario independently from it’s own file. I can execute the scenario-specific file that was imported in main.js. This is more useful when I don’t want to configure all the environment variables, but simply run the single test.

import http from 'k6/http';
import { check } from 'k6';
export { setup } from '../helpers.js';

export const options = {
  maxRedirects: 0,
  scenarios: {
    constantLoadScenario: {
      gracefulStop: '30s',
      vus: 1,
      executor: 'shared-iterations',
      iterations: 1
    }
  }
};

export default function (data) {
  cacheCreateAll(data);
}

function cacheCreateAll(data) {
  const createCache = http.post(`http://caches-service/test/all/caches`);

  check(
    createCache,
    {
      'is status 204': (r) => r.status === 204
    },
    {
      critical: 'critical'
    }
  );
}