Bug 1275501 - Generate docker images automatically when pushing r=fra…

…nziskus

automation/taskcluster/decision_task.yml

@@ -64,7 +64,6 @@ tasks:
           TC_SOURCE: {{{source}}}
           TC_PROJECT: {{project}}
           TC_COMMENT: '{{comment}}'
-          TC_IMAGE: "ttaubert/nss-ci:0.0.22"
           NSS_PUSHLOG_ID: '{{pushlog_id}}'
           NSS_HEAD_REPOSITORY: '{{{url}}}'
           NSS_HEAD_REVISION: '{{revision}}'

automation/taskcluster/graph/build.js

@@ -5,10 +5,12 @@
 var fs = require("fs");
 var path = require("path");
 var merge = require("merge");
-var yaml = require("js-yaml");
 var slugid = require("slugid");
 var flatmap = require("flatmap");
+
+var yaml = require("./yaml");
 var try_syntax = require("./try_syntax");
+var image_builder = require("./image_builder");
 
 // Default values for debugging.
 var TC_OWNER = process.env.TC_OWNER || "{{tc_owner}}";
@@ -18,49 +20,6 @@ var TC_COMMENT = process.env.TC_COMMENT || "{{tc_comment}}";
 var NSS_PUSHLOG_ID = process.env.NSS_PUSHLOG_ID || "{{nss_pushlog_id}}";
 var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}";
 
-// Register custom YAML types.
-var YAML_SCHEMA = yaml.Schema.create([
-  // Point in time at $now + x hours.
-  new yaml.Type('!from_now', {
-    kind: "scalar",
-
-    resolve: function (data) {
-      return true;
-    },
-
-    construct: function (data) {
-      var d = new Date();
-      d.setHours(d.getHours() + (data|0));
-      return d.toJSON();
-    }
-  }),
-
-  // Environment variables.
-  new yaml.Type('!env', {
-    kind: "scalar",
-
-    resolve: function (data) {
-      return true;
-    },
-
-    construct: function (data) {
-      return process.env[data] || "{{" + data.toLowerCase() + "}}";
-    }
-  })
-]);
-
-// Parse a given YAML file.
-function parseYamlFile(file, fallback) {
-  // Return fallback if the file doesn't exist.
-  if (!fs.existsSync(file) && fallback) {
-    return fallback;
-  }
-
-  // Otherwise, read the file or fail.
-  var source = fs.readFileSync(file, "utf-8");
-  return yaml.load(source, {schema: YAML_SCHEMA});
-}
-
 // Add base information to the given task.
 function decorateTask(task) {
   // Assign random task id.
@@ -78,10 +37,10 @@ function generateBuildTasks(platform, file) {
   var dir = path.join(__dirname, "./" + platform);
 
   // Parse base definitions.
-  var buildBase = parseYamlFile(path.join(dir, "_build_base.yml"), {});
-  var testBase = parseYamlFile(path.join(dir, "_test_base.yml"), {});
+  var buildBase = yaml.parse(path.join(dir, "_build_base.yml"), {});
+  var testBase = yaml.parse(path.join(dir, "_test_base.yml"), {});
 
-  return flatmap(parseYamlFile(path.join(dir, file)), function (task) {
+  return flatmap(yaml.parse(path.join(dir, file)), function (task) {
     // Merge base build task definition with the current one.
     var tasks = [task = merge.recursive(true, buildBase, task)];
 
@@ -120,7 +79,7 @@ function generateBuildTasks(platform, file) {
 function generateTestTasks(name, base, task) {
   // Load test definitions.
   var dir = path.join(__dirname, "./tests");
-  var tests = parseYamlFile(path.join(dir, name + ".yml"));
+  var tests = yaml.parse(path.join(dir, name + ".yml"));
 
   return tests.map(function (test) {
     // Merge test with base definition.
@@ -146,8 +105,8 @@ function generateTestTasks(name, base, task) {
 // Generate all tasks for a given platform.
 function generatePlatformTasks(platform) {
   var dir = path.join(__dirname, "./" + platform);
-  var buildBase = parseYamlFile(path.join(dir, "_build_base.yml"), {});
-  var testBase = parseYamlFile(path.join(dir, "_test_base.yml"), {});
+  var buildBase = yaml.parse(path.join(dir, "_build_base.yml"), {});
+  var testBase = yaml.parse(path.join(dir, "_test_base.yml"), {});
 
   // Parse all build tasks.
   return flatmap(fs.readdirSync(dir), function (file) {
@@ -179,5 +138,8 @@ if (TC_PROJECT == "nss-try") {
   graph.tasks = try_syntax.filterTasks(graph.tasks, TC_COMMENT);
 }
 
+// Inject the image builder tasks and dependencies.
+graph.tasks = image_builder.tweakTasks(graph.tasks);
+
 // Output the final graph.
 process.stdout.write(JSON.stringify(graph, null, 2));

automation/taskcluster/graph/image_builder.js

@@ -0,0 +1,98 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var fs = require("fs");
+var path = require("path");
+var crypto = require("crypto");
+var slugid = require("slugid");
+var flatmap = require("flatmap");
+
+var yaml = require("./yaml");
+
+// Default values for debugging.
+var TC_PROJECT = process.env.TC_PROJECT || "{{tc_project}}";
+var NSS_PUSHLOG_ID = process.env.NSS_PUSHLOG_ID || "{{nss_pushlog_id}}";
+var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}";
+
+// Add base information to the given task.
+function decorateTask(task) {
+  // Assign random task id.
+  task.taskId = slugid.v4();
+
+  // TreeHerder routes.
+  task.task.routes = [
+    "tc-treeherder-stage.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID,
+    "tc-treeherder.v2." + TC_PROJECT + "." + NSS_HEAD_REVISION + "." + NSS_PUSHLOG_ID
+  ];
+}
+
+// Compute the SHA-256 digest.
+function sha256(data) {
+  var hash = crypto.createHash("sha256");
+  hash.update(data);
+  return hash.digest("hex");
+}
+
+// Recursively collect a list of all files of a given directory.
+function collectFilesInDirectory(dir) {
+  return flatmap(fs.readdirSync(dir), function (entry) {
+    var entry_path = path.join(dir, entry);
+
+    if (fs.lstatSync(entry_path).isDirectory()) {
+      return collectFilesInDirectory(entry_path);
+    }
+
+    return [entry_path];
+  });
+}
+
+// Compute a hash over the given directory's contents.
+function hashDirectory(dir) {
+  var files = collectFilesInDirectory(dir).sort();
+  var hashes = files.map(function (file) {
+    return sha256(file + "|" + fs.readFileSync(file, "utf-8"));
+  });
+
+  return sha256(hashes.join(","));
+}
+
+// Generates the image-builder task description.
+function generateImageBuilderTask(context_path) {
+  var root = path.join(__dirname, "../../..");
+  var task = yaml.parse(path.join(__dirname, "image_builder.yml"), {});
+
+  // Add base info.
+  decorateTask(task);
+
+  // Add info for docker image building.
+  task.task.payload.env.CONTEXT_PATH = context_path;
+  task.task.payload.env.HASH = hashDirectory(path.join(root, context_path));
+
+  return task;
+}
+
+// Tweak the given list of tasks by injecting the image-builder task
+// and setting the right dependencies where needed.
+function tweakTasks(tasks) {
+  var id = "automation/taskcluster/docker";
+  var builder_task = generateImageBuilderTask(id);
+
+  tasks.forEach(function (task) {
+    if (task.task.payload.image == id) {
+      task.task.payload.image = {
+        taskId: builder_task.taskId,
+        path: "public/image.tar",
+        type: "task-image"
+      };
+
+      if (!task.requires) {
+        task.requires = [builder_task.taskId];
+      }
+    }
+  });
+
+  return [builder_task].concat(tasks);
+}
+
+module.exports.tweakTasks = tweakTasks;

automation/taskcluster/graph/image_builder.yml

@@ -0,0 +1,49 @@
+---
+reruns: 2
+
+task:
+  created: !from_now 0
+  deadline: !from_now 24
+  provisionerId: aws-provisioner-v1
+  workerType: hg-worker
+  schedulerId: task-graph-scheduler
+
+  metadata:
+    name: Image Builder
+    description: Image Builder
+    owner: !env TC_OWNER
+    source: !env TC_SOURCE
+
+  payload:
+    maxRunTime: 3600
+    image: taskcluster/image_builder:0.1.5
+
+    artifacts:
+      public/image.tar:
+        type: file
+        path: /artifacts/image.tar
+        expires: !from_now 8760
+
+    command:
+      - "/bin/bash"
+      - "-c"
+      - "/home/worker/bin/build_image.sh"
+
+    env:
+      HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
+      BASE_REPOSITORY: !env NSS_HEAD_REPOSITORY
+      HEAD_REV: !env NSS_HEAD_REVISION
+      HEAD_REF: !env NSS_HEAD_REVISION
+      PROJECT: !env TC_PROJECT
+
+    features:
+      dind: true
+
+  extra:
+    treeherder:
+      build:
+        platform: nss-decision
+      machine:
+        platform: nss-decision
+      jobKind: build
+      symbol: I

automation/taskcluster/graph/linux/_build_base.yml

@@ -14,7 +14,7 @@ task:
 
   payload:
     maxRunTime: 3600
-    image: !env TC_IMAGE
+    image: automation/taskcluster/docker
 
     artifacts:
       public:

automation/taskcluster/graph/linux/_test_base.yml

@@ -14,7 +14,7 @@ task:
 
   payload:
     maxRunTime: 3600
-    image: !env TC_IMAGE
+    image: automation/taskcluster/docker
 
     command:
       - "/bin/bash"

automation/taskcluster/graph/tools/_build_base.yml

@@ -14,7 +14,7 @@ task:
 
   payload:
     maxRunTime: 3600
-    image: !env TC_IMAGE
+    image: automation/taskcluster/docker
 
     env:
       NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY

automation/taskcluster/graph/yaml.js

@@ -0,0 +1,51 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var fs = require("fs");
+var yaml = require("js-yaml");
+
+// Register custom YAML types.
+var YAML_SCHEMA = yaml.Schema.create([
+  // Point in time at $now + x hours.
+  new yaml.Type('!from_now', {
+    kind: "scalar",
+
+    resolve: function (data) {
+      return true;
+    },
+
+    construct: function (data) {
+      var d = new Date();
+      d.setHours(d.getHours() + (data|0));
+      return d.toJSON();
+    }
+  }),
+
+  // Environment variables.
+  new yaml.Type('!env', {
+    kind: "scalar",
+
+    resolve: function (data) {
+      return true;
+    },
+
+    construct: function (data) {
+      return process.env[data] || "{{" + data.toLowerCase() + "}}";
+    }
+  })
+]);
+
+// Parse a given YAML file.
+function parse(file, fallback) {
+  // Return fallback if the file doesn't exist.
+  if (!fs.existsSync(file) && fallback) {
+    return fallback;
+  }
+
+  // Otherwise, read the file or fail.
+  var source = fs.readFileSync(file, "utf-8");
+  return yaml.load(source, {schema: YAML_SCHEMA});
+}
+
+module.exports.parse = parse;