Skip to content

Commit

Permalink
Bug 1274350 - Rewrite decision task generation to handle multiple pla…
Browse files Browse the repository at this point in the history 
…tforms better r=me

--HG--
rename : automation/taskcluster/graph/builds/linux32-debug.yml => automation/taskcluster/graph/linux/build32-debug.yml
rename : automation/taskcluster/graph/builds/linux32-opt.yml => automation/taskcluster/graph/linux/build32-opt.yml
rename : automation/taskcluster/graph/builds/linux64-asan.yml => automation/taskcluster/graph/linux/build64-asan.yml
rename : automation/taskcluster/graph/builds/linux64-debug.yml => automation/taskcluster/graph/linux/build64-debug.yml
rename : automation/taskcluster/graph/builds/linux64-opt.yml => automation/taskcluster/graph/linux/build64-opt.yml
rename : automation/taskcluster/graph/tasks/memleak.yml => automation/taskcluster/graph/tests/memleak.yml
rename : automation/taskcluster/graph/tasks/ssl_cycles.yml => automation/taskcluster/graph/tests/ssl.yml
  • Loading branch information
Tim Taubert committed Jun 2, 2016
1 parent 6e5bc6c commit 40c9910
Show file tree
Hide file tree
Showing 60 changed files with 675 additions and 990 deletions.
6 changes: 0 additions & 6 deletions .taskcluster.yml
View file
Expand Up @@ -71,15 +71,9 @@ tasks:
TC_SOURCE: {{{source}}}
TC_REVISION: '{{revision}}'
TC_REVISION_HASH: '{{revision_hash}}'
TC_DOCKER_IMAGE: "ttaubert/nss-ci:0.0.16"
TC_PROVISIONER_ID: "aws-provisioner-v1"
TC_WORKER_TYPE: "hg-worker"
NSS_HEAD_REPOSITORY: '{{{url}}}'
NSS_HEAD_REVISION: '{{revision}}'

features:
taskclusterProxy: true

maxRunTime: 1800

command:
Expand Down
242 changes: 110 additions & 132 deletions automation/taskcluster/graph/build.js
View file
Expand Up @@ -9,27 +9,17 @@ var yaml = require("js-yaml");
var slugid = require("slugid");
var flatmap = require("flatmap");

var TC_WORKER_TYPE = process.env.TC_WORKER_TYPE || "hg-worker";
var TC_PROVISIONER_ID = process.env.TC_PROVISIONER_ID || "aws-provisioner-v1";

// Default values for debugging.
var TC_REVISION = process.env.TC_REVISION || "{{tc_rev}}";
var TC_REVISION_HASH = process.env.TC_REVISION_HASH || "{{tc_rev_hash}}";
var TC_DOCKER_IMAGE = process.env.TC_DOCKER_IMAGE || "{{tc_docker_img}}";
var TC_OWNER = process.env.TC_OWNER || "{{tc_owner}}";
var TC_SOURCE = process.env.TC_SOURCE || "{{tc_source}}";
var NSS_HEAD_REPOSITORY = process.env.NSS_HEAD_REPOSITORY || "{{nss_head_repo}}";
var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}";

// Point in time at $now + x hours.
function from_now(hours) {
var d = new Date();
d.setHours(d.getHours() + (hours || 0));
return d.toJSON();
}

// Register custom YAML types.
var YAML_SCHEMA = yaml.Schema.create([
// Point in time at $now + x hours.
new yaml.Type('!from_now', {
kind: "scalar",

Expand All @@ -38,149 +28,137 @@ var YAML_SCHEMA = yaml.Schema.create([
},

construct: function (data) {
return from_now(data|0);
var d = new Date();
d.setHours(d.getHours() + (data|0));
return d.toJSON();
}
})
]);
}),

// Parse a directory containing YAML files.
function parseDirectory(dir) {
var tasks = {};
// Environment variables.
new yaml.Type('!env', {
kind: "scalar",

resolve: function (data) {
return true;
},

fs.readdirSync(dir).forEach(function (file) {
if (file.endsWith(".yml")) {
var source = fs.readFileSync(path.join(dir, file), "utf-8");
tasks[file.slice(0, -4)] = yaml.load(source, {schema: YAML_SCHEMA});
construct: function (data) {
return process.env[data];
}
});
})
]);

return tasks;
// Parse a given YAML file.
function parseYamlFile(file, fallback) {
// Return fallback if the file doesn't exist.
if (!fs.existsSync(file) && fallback) {
return fallback;
}

// Otherwise, read the file or fail.
var source = fs.readFileSync(file, "utf-8");
return yaml.load(source, {schema: YAML_SCHEMA});
}

// Generates a task using a given definition.
function generateTasks(definition) {
var task = {
taskId: slugid.v4(),
reruns: 2,

task: task = {
created: from_now(0),
deadline: from_now(24),
provisionerId: TC_PROVISIONER_ID,
workerType: TC_WORKER_TYPE,
schedulerId: "task-graph-scheduler",

scopes: [
"queue:route:tc-treeherder-stage.nss." + TC_REVISION_HASH,
"queue:route:tc-treeherder.nss." + TC_REVISION_HASH,
"scheduler:extend-task-graph:*"
],

routes: [
"tc-treeherder-stage.nss." + TC_REVISION_HASH,
"tc-treeherder.nss." + TC_REVISION_HASH
],

metadata: {
owner: TC_OWNER,
source: TC_SOURCE
},

payload: {
image: TC_DOCKER_IMAGE,
maxRunTime: 3600,

env: {
NSS_HEAD_REPOSITORY: NSS_HEAD_REPOSITORY,
NSS_HEAD_REVISION: NSS_HEAD_REVISION
}
},
// Generate all tasks for a given build.
function generateBuildTasks(platform, file) {
var dir = path.join(__dirname, "./" + platform);

extra: {
treeherder: {
revision: TC_REVISION,
revision_hash: TC_REVISION_HASH
}
}
}
};
// Parse base definitions.
var buildBase = parseYamlFile(path.join(dir, "_build_base.yml"), {});
var testBase = parseYamlFile(path.join(dir, "_test_base.yml"), {});

// Merge base task definition with the YAML one.
var tasks = [task = merge.recursive(true, task, definition)];
return flatmap(parseYamlFile(path.join(dir, file)), function (task) {
// Merge base build task definition with the current one.
var tasks = [task = merge.recursive(true, buildBase, task)];

// Generate dependent tasks.
if (task.dependents) {
// The base definition for all subtasks.
var base = {
requires: [task.taskId],
// Assign random task id.
task.taskId = slugid.v4();

task: {
payload: {
env: {
TC_PARENT_TASK_ID: task.taskId
// Generate test tasks.
if (task.tests) {
// The base definition for all tests of this platform.
var base = merge.recursive(true, {
requires: [task.taskId],

task: {
payload: {
env: {
TC_PARENT_TASK_ID: task.taskId
}
}
}
}
};

// We clone everything but the taskId, we need a new and unique one.
delete base.taskId;

// Iterate and generate all subtasks.
var subtasks = flatmap(task.dependents, function (name) {
if (!(name in TASKS)) {
throw new Error("Can't find task '" + name + "'");
}

return flatmap(TASKS[name], function (subtask) {
// Merge subtask with base definition.
var dependent = merge.recursive(true, subtask, base);

// We only want to carry over environment variables and
// TreeHerder configuration data.
dependent.task.payload.env =
merge.recursive(true, task.task.payload.env,
dependent.task.payload.env);
dependent.task.extra.treeherder =
merge.recursive(true, task.task.extra.treeherder,
dependent.task.extra.treeherder);

// Print all subtasks.
return generateTasks(dependent);
});
});
}, testBase);

// Append subtasks.
tasks = tasks.concat(subtasks);
// Generate and append test task definitions.
tasks = tasks.concat(flatmap(task.tests, function (name) {
return generateTestTasks(name, base, task);
}));

// The dependents field is not part of the schema.
delete task.dependents;
}
// |tests| is not part of the schema.
delete task.tests;
}

// Convert env variables to strings.
tasks.forEach(function (task) {
var env = task.task.payload.env || {};
Object.keys(env).forEach(function (name) {
if (typeof(env[name]) != "undefined") {
env[name] = env[name] + "";
}
});
return tasks;
});
}

// Generate all tasks for a given test.
function generateTestTasks(name, base, task) {
// Load test definitions.
var dir = path.join(__dirname, "./tests");
var tests = parseYamlFile(path.join(dir, name + ".yml"));

return tests.map(function (test) {
// Merge test with base definition.
test = merge.recursive(true, base, test);

// Assign random task id.
test.taskId = slugid.v4();

// We only want to carry over environment variables...
test.task.payload.env =
merge.recursive(true, task.task.payload.env,
test.task.payload.env);

return tasks;
// ...and TreeHerder configuration data.
test.task.extra.treeherder =
merge.recursive(true, task.task.extra.treeherder,
test.task.extra.treeherder);

return test;
});
}

// Parse YAML task definitions.
var BUILDS = parseDirectory(path.join(__dirname, "./builds/"));
var TASKS = parseDirectory(path.join(__dirname, "./tasks/"));
// Generate all tasks for a given platform.
function generatePlatformTasks(platform) {
var dir = path.join(__dirname, "./" + platform);
var buildBase = parseYamlFile(path.join(dir, "_build_base.yml"), {});
var testBase = parseYamlFile(path.join(dir, "_test_base.yml"), {});

// Parse all build tasks.
return flatmap(fs.readdirSync(dir), function (file) {
if (!file.startsWith("_") && file.endsWith(".yml")) {
var tasks = generateBuildTasks(platform, file);

// Convert env variables to strings.
tasks.forEach(function (task) {
var env = task.task.payload.env || {};
Object.keys(env).forEach(function (name) {
if (typeof(env[name]) != "undefined") {
env[name] = env[name] + "";
}
});
});

return tasks;
}
});
}

// Construct the task graph.
var graph = {
// Use files in the "builds" directory as roots.
tasks: flatmap(Object.keys(BUILDS), function (name) {
return flatmap(BUILDS[name], function (build) {
return generateTasks(build);
});
})
tasks: flatmap(["linux", "windows", "tools"], generatePlatformTasks)
};

// Output the final graph.
Expand Down
18 changes: 0 additions & 18 deletions automation/taskcluster/graph/builds/clang-format.yml
View file

This file was deleted.

32 changes: 0 additions & 32 deletions automation/taskcluster/graph/builds/linux64-memleak.yml
View file

This file was deleted.

39 changes: 0 additions & 39 deletions automation/taskcluster/graph/builds/win2012x64-debug.yml
View file

This file was deleted.

0 comments on commit 40c9910

Please sign in to comment.