Skip to content

Commit 4496068

Browse files
ahalahal
committed
feat: add scriptworker signing transforms
1 parent 46fdc27 commit 4496068

5 files changed

Lines changed: 467 additions & 0 deletions

File tree

‎src/mozilla_taskgraph/transforms/scriptworker/signing.py‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# This Source Code Form is subject to the terms of the Mozilla Public
2+
# License, v. 2.0. If a copy of the MPL was not distributed with this
3+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
import re
6+
7+
from taskgraph.transforms.base import TransformSequence
8+
from taskgraph.util.schema import (
9+
Schema,
10+
optionally_keyed_by,
11+
resolve_keyed_by,
12+
)
13+
from voluptuous import ALLOW_EXTRA, Any, Optional, Required
14+
15+
SIGNING_FORMATS = ["autograph_gpg"]
16+
SIGNING_TYPES = ["dep", "release"]
17+
DETACHED_SIGNATURE_EXTENSION = ".asc"
18+
19+
signing_schema = Schema(
20+
{
21+
Required("attributes"): {
22+
Optional("artifacts"): dict,
23+
Required("build-type"): str,
24+
},
25+
Required("signing"): optionally_keyed_by(
26+
"build-type",
27+
"level",
28+
{
29+
Required("format"): optionally_keyed_by(
30+
"build-type", "level", Any(*SIGNING_FORMATS)
31+
),
32+
Optional("type"): optionally_keyed_by(
33+
"build-type", "level", Any(*SIGNING_TYPES)
34+
),
35+
Optional("ignore-artifacts"): list,
36+
},
37+
),
38+
Required("worker"): {
39+
Required("upstream-artifacts"): [
40+
{
41+
# Paths to the artifacts to sign
42+
Required("paths"): [str],
43+
}
44+
],
45+
},
46+
},
47+
extra=ALLOW_EXTRA,
48+
)
49+
50+
transforms = TransformSequence()
51+
transforms.add_validate(signing_schema)
52+
53+
54+
@transforms.add
55+
def resolve_signing_keys(config, tasks):
56+
for task in tasks:
57+
for key in (
58+
"signing",
59+
"signing.format",
60+
"signing.type",
61+
):
62+
resolve_keyed_by(
63+
task,
64+
key,
65+
item_name=task["name"],
66+
**{
67+
"build-type": task["attributes"]["build-type"],
68+
"level": config.params["level"],
69+
},
70+
)
71+
yield task
72+
73+
74+
@transforms.add
75+
def set_signing_attributes(_, tasks):
76+
for task in tasks:
77+
task["attributes"]["signed"] = True
78+
yield task
79+
80+
81+
@transforms.add
82+
def set_signing_format(_, tasks):
83+
for task in tasks:
84+
for upstream_artifact in task["worker"]["upstream-artifacts"]:
85+
upstream_artifact["formats"] = [task["signing"]["format"]]
86+
yield task
87+
88+
89+
@transforms.add
90+
def set_signing_and_worker_type(config, tasks):
91+
for task in tasks:
92+
signing_type = task["signing"].get("type")
93+
if not signing_type:
94+
signing_type = "release" if config.params["level"] == "3" else "dep"
95+
96+
task.setdefault("worker", {})["signing-type"] = f"{signing_type}-signing"
97+
98+
if "worker-type" not in task:
99+
worker_type = "signing"
100+
build_type = task["attributes"]["build-type"]
101+
102+
if signing_type == "dep":
103+
worker_type = f"dep-{worker_type}"
104+
if build_type == "macos":
105+
worker_type = f"{build_type}-{worker_type}"
106+
task["worker-type"] = worker_type
107+
108+
yield task
109+
110+
111+
@transforms.add
112+
def filter_out_ignored_artifacts(_, tasks):
113+
for task in tasks:
114+
ignore = task["signing"].get("ignore-artifacts")
115+
if not ignore:
116+
yield task
117+
continue
118+
119+
def is_ignored(artifact):
120+
return not any(re.search(i, artifact) for i in ignore)
121+
122+
if task["attributes"].get("artifacts"):
123+
task["attributes"]["artifacts"] = {
124+
extension: path
125+
for extension, path in task["attributes"]["artifacts"].items()
126+
if is_ignored(path)
127+
}
128+
129+
for upstream_artifact in task["worker"]["upstream-artifacts"]:
130+
upstream_artifact["paths"] = [
131+
path for path in upstream_artifact["paths"] if is_ignored(path)
132+
]
133+
134+
yield task
135+
136+
137+
@transforms.add
138+
def set_gpg_detached_signature_artifacts(_, tasks):
139+
for task in tasks:
140+
if task["signing"]["format"] != "autograph_gpg":
141+
yield task
142+
continue
143+
144+
task["attributes"]["artifacts"] = {
145+
extension
146+
+ DETACHED_SIGNATURE_EXTENSION: path
147+
+ DETACHED_SIGNATURE_EXTENSION
148+
for extension, path in task["attributes"]["artifacts"].items()
149+
}
150+
151+
yield task
152+
153+
154+
@transforms.add
155+
def remove_signing_config(_, tasks):
156+
for task in tasks:
157+
del task["signing"]
158+
yield task

‎taskcluster/test/params/main-push.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
---
2+
base_ref: refs/heads/main
3+
base_repository: https://github.com/mozilla-releng/mozilla-taskgraph
4+
base_rev: a76ea4308313211a99e8e501c5a97a5ce2c08cc1
5+
build_date: 1681151087
6+
build_number: 1
7+
do_not_optimize: []
8+
enable_always_target: true
9+
existing_tasks: {}
10+
filters:
11+
- target_tasks_method
12+
head_ref: refs/heads/main
13+
head_repository: https://github.com/mozilla-releng/mozilla-taskgraph
14+
head_rev: a0785edae4a841b6119925280c744000f59b903e
15+
head_tag: ''
16+
level: '1'
17+
moz_build_date: '20230410182447'
18+
next_version: null
19+
optimize_strategies: null
20+
optimize_target_tasks: true
21+
owner: ahal@pm.me
22+
project: mozilla-taskgraph
23+
pushdate: 0
24+
pushlog_id: '0'
25+
repository_type: git
26+
target_tasks_method: default
27+
tasks_for: github-push
28+
version: null

‎taskcluster/test/params/pull-request.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
---
2+
base_ref: main
3+
base_repository: https://github.com/mozilla-releng/mozilla-taskgraph
4+
base_rev: a0785edae4a841b6119925280c744000f59b903e
5+
build_date: 1681154438
6+
build_number: 1
7+
do_not_optimize: []
8+
enable_always_target: true
9+
existing_tasks: {}
10+
filters:
11+
- target_tasks_method
12+
head_ref: codecov
13+
head_repository: https://github.com/user/mozilla-taskgraph
14+
head_rev: 06c766e8e9d558eed5ccf8029164120a27af5fb1
15+
head_tag: ''
16+
level: '1'
17+
moz_build_date: '20230410192038'
18+
next_version: null
19+
optimize_strategies: null
20+
optimize_target_tasks: true
21+
owner: user@example.com
22+
project: mozilla-taskgraph
23+
pushdate: 0
24+
pushlog_id: '0'
25+
repository_type: git
26+
target_tasks_method: default
27+
tasks_for: github-pull-request
28+
version: null

‎test/conftest.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
from pathlib import Path
2+
3+
import pytest
4+
from taskgraph.config import GraphConfig
5+
from taskgraph.transforms.base import TransformConfig
6+
7+
here = Path(__file__).parent
8+
9+
10+
@pytest.fixture(scope="session")
11+
def datadir():
12+
return here / "data"
13+
14+
15+
def fake_load_graph_config(root_dir):
16+
graph_config = GraphConfig(
17+
{
18+
"trust-domain": "test-domain",
19+
"taskgraph": {
20+
"repositories": {
21+
"ci": {"name": "Taskgraph"},
22+
}
23+
},
24+
"workers": {
25+
"aliases": {
26+
"b-linux": {
27+
"provisioner": "taskgraph-b",
28+
"implementation": "docker-worker",
29+
"os": "linux",
30+
"worker-type": "linux",
31+
},
32+
"t-linux": {
33+
"provisioner": "taskgraph-t",
34+
"implementation": "docker-worker",
35+
"os": "linux",
36+
"worker-type": "linux",
37+
},
38+
}
39+
},
40+
"task-priority": "low",
41+
"treeherder": {"group-names": {"T": "tests"}},
42+
},
43+
root_dir,
44+
)
45+
graph_config.__dict__["register"] = lambda: None
46+
return graph_config
47+
48+
49+
@pytest.fixture
50+
def graph_config(datadir):
51+
return fake_load_graph_config(str(datadir / "taskcluster" / "ci"))
52+
53+
54+
class FakeParameters(dict):
55+
strict = True
56+
57+
def is_try(self):
58+
return False
59+
60+
def file_url(self, path, pretty=False):
61+
return path
62+
63+
64+
@pytest.fixture
65+
def parameters():
66+
return FakeParameters(
67+
{
68+
"base_repository": "http://hg.example.com",
69+
"build_date": 0,
70+
"build_number": 1,
71+
"enable_always_target": True,
72+
"head_repository": "http://hg.example.com",
73+
"head_rev": "abcdef",
74+
"head_ref": "default",
75+
"level": "1",
76+
"moz_build_date": 0,
77+
"next_version": "1.0.1",
78+
"owner": "some-owner",
79+
"project": "some-project",
80+
"pushlog_id": 1,
81+
"repository_type": "hg",
82+
"target_tasks_method": "test_method",
83+
"tasks_for": "hg-push",
84+
"try_mode": None,
85+
"version": "1.0.0",
86+
}
87+
)
88+
89+
90+
@pytest.fixture
91+
def make_transform_config(parameters, graph_config):
92+
def inner(kind_config=None, kind_dependencies_tasks=None):
93+
kind_config = kind_config or {}
94+
kind_dependencies_tasks = kind_dependencies_tasks or {}
95+
return TransformConfig(
96+
"test",
97+
str(here),
98+
kind_config,
99+
parameters,
100+
kind_dependencies_tasks,
101+
graph_config,
102+
write_artifacts=False,
103+
)
104+
105+
return inner
106+
107+
108+
@pytest.fixture
109+
def run_transform(make_transform_config):
110+
def inner(func, tasks, config=None):
111+
if not isinstance(tasks, list):
112+
tasks = [tasks]
113+
114+
if not config:
115+
config = make_transform_config()
116+
return list(func(config, tasks))
117+
118+
return inner

0 commit comments

Comments
 (0)