From c388b79c16bc1f5bde9dcca80eaf1d78a5a1e47a Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 31 Oct 2024 08:42:27 +1300 Subject: [PATCH 1/6] Move functions into a helper module --- helpers/__init__.py | 0 helpers/github.py | 214 ++++++++++++++++++++++++++ helpers/zenhub.py | 151 ++++++++++++++++++ zcash-issue-dag.py | 361 ++------------------------------------------ 4 files changed, 375 insertions(+), 351 deletions(-) create mode 100644 helpers/__init__.py create mode 100644 helpers/github.py create mode 100644 helpers/zenhub.py diff --git a/helpers/__init__.py b/helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/helpers/github.py b/helpers/github.py new file mode 100644 index 000000000..1c701ba5b --- /dev/null +++ b/helpers/github.py @@ -0,0 +1,214 @@ +from sgqlc.endpoint.http import HTTPEndpoint +from sgqlc.operation import Operation + +from github_schema import github_schema as schema + +# To get the id of a repo, see . + +HALO2_REPOS = { + 290019239: ('zcash', 'halo2'), + 344239327: ('zcash', 'pasta_curves'), +} + +CORE_REPOS = { + 26987049: ('zcash', 'zcash'), + 47279130: ('zcash', 'zips'), + 48303644: ('zcash', 'incrementalmerkletree'), + 85334928: ('zcash', 'librustzcash'), + 133857578: ('zcash-hackworks', 'zcash-test-vectors'), + 111058300: ('zcash', 'sapling-crypto'), + **HALO2_REPOS, + 305835578: ('zcash', 'orchard'), +} + +TFL_REPOS = { + 642135348: ('Electric-Coin-Company', 'tfl-book'), + 725179873: ('Electric-Coin-Company', 'zebra-tfl'), + 695805989: ('zcash', 'simtfl'), +} + +ANDROID_REPOS = { + 390808594: ('Electric-Coin-Company', 'zashi-android'), + 151763639: ('Electric-Coin-Company', 'zcash-android-wallet-sdk'), + 719178328: ('Electric-Coin-Company', 'zashi'), +} + +IOS_REPOS = { + 387551125: ('Electric-Coin-Company', 'zashi-ios'), + 185480114: ('Electric-Coin-Company', 'zcash-swift-wallet-sdk'), + 270825987: ('Electric-Coin-Company', 'MnemonicSwift'), + 439137887: ('Electric-Coin-Company', 'zcash-light-client-ffi'), + 719178328: ('Electric-Coin-Company', 'zashi'), +} + +WALLET_REPOS = { + 85334928: ('zcash', 'librustzcash'), + 159714694: ('zcash', 'lightwalletd'), + **ANDROID_REPOS, + **IOS_REPOS, +} + +ECC_REPOS = { + **CORE_REPOS, + **TFL_REPOS, + **WALLET_REPOS, + 65419597: ('Electric-Coin-Company', 'infrastructure'), +} + +ZF_REPOS = { + 205255683: ('ZcashFoundation', 'zebra'), + 225479018: ('ZcashFoundation', 'redjubjub'), + 235651437: ('ZcashFoundation', 'ed25519-zebra'), + 279422254: ('ZcashFoundation', 'zcash_script'), +} + +ZF_FROST_REPOS = { + 437862440: ('ZcashFoundation', 'frost'), +} + +ZCASHD_DEPRECATION_REPOS = { + 26987049: ('zcash', 'zcash'), + 47279130: ('zcash', 'zips'), + 85334928: ('zcash', 'librustzcash'), + 863610221: ('zcash', 'wallet'), + 159714694: ('zcash', 'lightwalletd'), +} + +POOL_DEPRECATION_REPOS = { + **CORE_REPOS, + **WALLET_REPOS, +} + +REPO_SETS = { + 'core': CORE_REPOS, + 'halo2': HALO2_REPOS, + 'tfl': TFL_REPOS, + 'wallet': WALLET_REPOS, + 'wallet-ios': IOS_REPOS, + 'wallet-android': ANDROID_REPOS, + 'ecc': ECC_REPOS, + 'zf': ZF_REPOS, + 'zf-frost': ZF_FROST_REPOS, + 'zf-devops': {**ZF_REPOS, **ZF_FROST_REPOS}, + 'zcashd-deprecation': ZCASHD_DEPRECATION_REPOS, + 'sprout-deprecation': POOL_DEPRECATION_REPOS, + 'transparent-deprecation': POOL_DEPRECATION_REPOS, +} + + +def api(token): + return HTTPEndpoint( + 'https://api.github.com/graphql', + {'Authorization': 'bearer %s' % token}, + ) + + +class GitHubIssue: + def __init__(self, repo_id, issue_number, data, REPOS): + self.repo_id = repo_id + self.issue_number = issue_number + self.milestone = None + self._REPOS = REPOS + + if data is not None: + labels = [label['name'] for label in data['labels']['nodes']] + self.title = data['title'] + self.is_release = 'C-release' in labels + self.is_target = 'C-target' in labels + self.is_pr = 'merged' in data + self.is_committed = 'S-committed' in labels + self.is_in_progress = 'S-in-progress' in labels + self.waiting_on_review = 'S-waiting-on-review' in labels + self.url = data['url'] + self.state = 'closed' if data['state'] in ['CLOSED', 'MERGED'] else 'open' + if 'milestone' in data and data['milestone']: + self.milestone = data['milestone']['title'] + else: + # If we can't fetch issue data, assume we don't care. + self.title = '' + self.url = None + self.is_release = False + self.is_target = False + self.is_pr = False + self.is_committed = False + self.is_in_progress = False + self.waiting_on_review = False + self.state = 'closed' + + def __repr__(self): + if self.repo_id in self._REPOS: + repo = self._REPOS[self.repo_id] + # Shorten the representation of long repo names. + if repo[0] == 'Electric-Coin-Company': + repo = ('ECC', repo[1]) + repo = '/'.join(repo) + return '%s#%d' % (repo, self.issue_number) + else: + return 'Unknown' + + def __eq__(self, other): + return (self.repo_id, self.issue_number) == (other.repo_id, other.issue_number) + + def __hash__(self): + return hash((self.repo_id, self.issue_number)) + + def any_cat(self, categories): + release_cat = self.is_release if 'releases' in categories else False + targets_cat = self.is_target if 'targets' in categories else False + return release_cat or targets_cat + + +def fetch_issues(op, issues, REPOS): + repos = set([repo for (repo, _) in issues]) + repos = {repo: [issue for (r, issue) in issues if r == repo] for repo in repos} + + for repo, issues in repos.items(): + conn = op.repository( + owner=REPOS[repo][0], + name=REPOS[repo][1], + __alias__='repo%d' % repo, + ) + + for issue in issues: + res = conn.issue_or_pull_request(number=issue, __alias__='issue%d' % issue) + for typ in [schema.Issue, schema.PullRequest]: + node = res.__as__(typ) + node.labels(first=50).nodes().name() + node.state() + node.milestone().title() + node.title() + node.url() + if typ == schema.PullRequest: + node.merged() + + +def download_issues(endpoint, nodes, REPOS): + issues = [(repo, issue) for (repo, issue) in nodes if repo in REPOS] + + ret = {} + + # Ensure that any graph nodes from ZenHub that are not in the repos we care about have + # default entries, to simplify subsequent graph manipulation code. + for repo, issue in [(repo, issue) for (repo, issue) in nodes if repo not in REPOS]: + ret[(repo, issue)] = GitHubIssue(repo, issue, None, REPOS) + + def chunks(lst, n): + for i in range(0, len(lst), n): + yield lst[i : i + n] + + for issues in chunks(issues, 50): + op = Operation(schema.Query) + fetch_issues(op, issues, REPOS) + + d = endpoint(op) + data = op + d + + for repo, issue in issues: + repo_data = data['repo%d' % repo] + issue_key = 'issue%d' % issue + # If GITHUB_TOKEN doesn't have permission to read from a particular private + # repository in REPOS, GitHub returns an empty repo_data section. + issue_data = repo_data[issue_key] if issue_key in repo_data else None + ret[(repo, issue)] = GitHubIssue(repo, issue, issue_data, REPOS) + + return ret diff --git a/helpers/zenhub.py b/helpers/zenhub.py new file mode 100644 index 000000000..00ee46d3a --- /dev/null +++ b/helpers/zenhub.py @@ -0,0 +1,151 @@ +import networkx as nx +from sgqlc.endpoint.http import HTTPEndpoint +from sgqlc.operation import Operation + +from helpers.github import CORE_REPOS, WALLET_REPOS, ZF_REPOS, ZF_FROST_REPOS +from zenhub_schema import zenhub_schema + +WORKSPACE_SETS = { + # ecc-core + '5dc1fd615862290001229f21': CORE_REPOS.keys(), + # ecc-wallet + '5db8aa0244512d0001e0968e': WALLET_REPOS.keys(), + # zf + '5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(), + # zf-frost + '607d75e0169bd50011d5410f': ZF_FROST_REPOS.keys(), +} + + +def api(token): + return HTTPEndpoint( + 'https://api.zenhub.com/public/graphql', + {'Authorization': 'Bearer %s' % token}, + ) + + +def fetch_workspace_graph(op, workspace_id, repos, cursor): + dependencies = op.workspace(id=workspace_id).issue_dependencies( + # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here, + # not the GitHub repo IDs (which the previous REST API used). + # repository_ids=repos, + first=100, + after=cursor, + ) + dependencies.nodes.id() + dependencies.nodes.blocked_issue.number() + dependencies.nodes.blocked_issue.repository.gh_id() + dependencies.nodes.blocking_issue.number() + dependencies.nodes.blocking_issue.repository.gh_id() + dependencies.page_info.has_next_page() + dependencies.page_info.end_cursor() + + +def get_dependency_graph(endpoint, workspace_id, repos): + edges = [] + cursor = None + + while True: + op = Operation(zenhub_schema.Query) + fetch_workspace_graph(op, workspace_id, repos, cursor) + + d = endpoint(op) + data = op + d + + dependencies = data.workspace.issue_dependencies + edges += [ + ( + (node.blocking_issue.repository.gh_id, node.blocking_issue.number), + (node.blocked_issue.repository.gh_id, node.blocked_issue.number), + ) + for node in dependencies.nodes + ] + + if dependencies.page_info.has_next_page: + cursor = dependencies.page_info.end_cursor + print('.', end='', flush=True) + else: + print() + break + + return nx.DiGraph(edges) + + +def fetch_epics(op, workspace_id, repos, cursor): + epics = op.workspace(id=workspace_id).epics( + # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here, + # not the GitHub repo IDs (which the previous REST API used). + # repository_ids=repos, + first=100, + after=cursor, + ) + epics.nodes.id() + epics.nodes.issue.number() + epics.nodes.issue.repository.gh_id() + epics.page_info.has_next_page() + epics.page_info.end_cursor() + + +def get_epics(endpoint, workspace_id, repos): + epics = [] + cursor = None + + while True: + op = Operation(zenhub_schema.Query) + fetch_epics(op, workspace_id, repos, cursor) + + d = endpoint(op) + data = op + d + + epics_page = data.workspace.epics + epics += [ + (node.id, (node.issue.repository.gh_id, node.issue.number)) + for node in epics_page.nodes + ] + + if epics_page.page_info.has_next_page: + cursor = epics_page.page_info.end_cursor + print('.', end='', flush=True) + else: + print() + break + + return epics + + +def fetch_epic_issues(op, workspace_id, epic_id, cursor): + epic = op.workspace(id=workspace_id).epics(ids=[epic_id]) + child_issues = epic.nodes.child_issues( + first=100, + after=cursor, + ) + child_issues.nodes.number() + child_issues.nodes.repository.gh_id() + child_issues.page_info.has_next_page() + child_issues.page_info.end_cursor() + + +def get_epic_issues(endpoint, workspace_id, epic_id): + epic_issues = [] + cursor = None + + while True: + op = Operation(zenhub_schema.Query) + fetch_epic_issues(op, workspace_id, epic_id, cursor) + + d = endpoint(op) + data = op + d + + epic = data.workspace.epics.nodes[0] + epic_issues += [ + (node.repository.gh_id, node.number) for node in epic.child_issues.nodes + ] + + if epic.child_issues.page_info.has_next_page: + cursor = epic.child_issues.page_info.end_cursor + print('.', end='', flush=True) + else: + print() + break + + return epic_issues diff --git a/zcash-issue-dag.py b/zcash-issue-dag.py index 875a6e228..8a5e7a94c 100755 --- a/zcash-issue-dag.py +++ b/zcash-issue-dag.py @@ -7,129 +7,22 @@ import networkx as nx from str2bool import str2bool as strtobool -import mimetypes import os from textwrap import wrap from urllib.parse import urlparse -from sgqlc.endpoint.http import HTTPEndpoint -from sgqlc.operation import Operation -from github_schema import github_schema as schema -from zenhub_schema import zenhub_schema +from helpers import github, zenhub GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN') ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN') DAG_VIEW = os.environ.get('DAG_VIEW', 'core') -# To get the id of a repo, see . - -HALO2_REPOS = { - 290019239: ('zcash', 'halo2'), - 344239327: ('zcash', 'pasta_curves'), -} - -CORE_REPOS = { - 26987049: ('zcash', 'zcash'), - 47279130: ('zcash', 'zips'), - 48303644: ('zcash', 'incrementalmerkletree'), - 85334928: ('zcash', 'librustzcash'), - 133857578: ('zcash-hackworks', 'zcash-test-vectors'), - 111058300: ('zcash', 'sapling-crypto'), - **HALO2_REPOS, - 305835578: ('zcash', 'orchard'), -} - -TFL_REPOS = { - 642135348: ('Electric-Coin-Company', 'tfl-book'), - 725179873: ('Electric-Coin-Company', 'zebra-tfl'), - 695805989: ('zcash', 'simtfl'), -} - -ANDROID_REPOS = { - 390808594: ('Electric-Coin-Company', 'zashi-android'), - 151763639: ('Electric-Coin-Company', 'zcash-android-wallet-sdk'), - 719178328: ('Electric-Coin-Company', 'zashi'), -} - -IOS_REPOS = { - 387551125: ('Electric-Coin-Company', 'zashi-ios'), - 185480114: ('Electric-Coin-Company', 'zcash-swift-wallet-sdk'), - 270825987: ('Electric-Coin-Company', 'MnemonicSwift'), - 439137887: ('Electric-Coin-Company', 'zcash-light-client-ffi'), - 719178328: ('Electric-Coin-Company', 'zashi'), -} - -WALLET_REPOS = { - 85334928: ('zcash', 'librustzcash'), - 159714694: ('zcash', 'lightwalletd'), - **ANDROID_REPOS, - **IOS_REPOS, -} - -ECC_REPOS = { - **CORE_REPOS, - **TFL_REPOS, - **WALLET_REPOS, - 65419597: ('Electric-Coin-Company', 'infrastructure'), -} - -ZF_REPOS = { - 205255683: ('ZcashFoundation', 'zebra'), - 225479018: ('ZcashFoundation', 'redjubjub'), - 235651437: ('ZcashFoundation', 'ed25519-zebra'), - 279422254: ('ZcashFoundation', 'zcash_script'), -} - -ZF_FROST_REPOS = { - 437862440: ('ZcashFoundation', 'frost'), -} - -ZCASHD_DEPRECATION_REPOS = { - 26987049: ('zcash', 'zcash'), - 47279130: ('zcash', 'zips'), - 85334928: ('zcash', 'librustzcash'), - 863610221: ('zcash', 'wallet'), - 159714694: ('zcash', 'lightwalletd'), -} - -POOL_DEPRECATION_REPOS = { - **CORE_REPOS, - **WALLET_REPOS, -} - -REPO_SETS = { - 'core': CORE_REPOS, - 'halo2': HALO2_REPOS, - 'tfl': TFL_REPOS, - 'wallet': WALLET_REPOS, - 'wallet-ios': IOS_REPOS, - 'wallet-android': ANDROID_REPOS, - 'ecc': ECC_REPOS, - 'zf': ZF_REPOS, - 'zf-frost': ZF_FROST_REPOS, - 'zf-devops': {**ZF_REPOS, **ZF_FROST_REPOS}, - 'zcashd-deprecation': ZCASHD_DEPRECATION_REPOS, - 'sprout-deprecation': POOL_DEPRECATION_REPOS, - 'transparent-deprecation': POOL_DEPRECATION_REPOS, -} - -REPOS = REPO_SETS[DAG_VIEW] - -WORKSPACE_SETS = { - # ecc-core - '5dc1fd615862290001229f21': CORE_REPOS.keys(), - # ecc-wallet - '5db8aa0244512d0001e0968e': WALLET_REPOS.keys(), - # zf - '5fb24d9264a3e8000e666a9e': ZF_REPOS.keys(), - # zf-frost - '607d75e0169bd50011d5410f': ZF_FROST_REPOS.keys(), -} +REPOS = github.REPO_SETS[DAG_VIEW] WORKSPACES = { workspace_id: [repo_id for repo_id in repos if repo_id in REPOS] - for (workspace_id, repos) in WORKSPACE_SETS.items() + for (workspace_id, repos) in zenhub.WORKSPACE_SETS.items() } SUPPORTED_CATEGORIES = set(['releases', 'targets']) @@ -160,248 +53,14 @@ def cats(s): SHOW_EPICS = strtobool(os.environ.get('SHOW_EPICS', 'false')) -class GitHubIssue: - def __init__(self, repo_id, issue_number, data): - self.repo_id = repo_id - self.issue_number = issue_number - self.milestone = None - - if data is not None: - labels = [label['name'] for label in data['labels']['nodes']] - self.title = data['title'] - self.is_release = 'C-release' in labels - self.is_target = 'C-target' in labels - self.is_pr = 'merged' in data - self.is_committed = 'S-committed' in labels - self.is_in_progress = 'S-in-progress' in labels - self.waiting_on_review = 'S-waiting-on-review' in labels - self.url = data['url'] - self.state = 'closed' if data['state'] in ['CLOSED', 'MERGED'] else 'open' - if 'milestone' in data and data['milestone']: - self.milestone = data['milestone']['title'] - else: - # If we can't fetch issue data, assume we don't care. - self.title = '' - self.url = None - self.is_release = False - self.is_target = False - self.is_pr = False - self.is_committed = False - self.is_in_progress = False - self.waiting_on_review = False - self.state = 'closed' - - def __repr__(self): - if self.repo_id in REPOS: - repo = REPOS[self.repo_id] - # Shorten the representation of long repo names. - if repo[0] == 'Electric-Coin-Company': - repo = ('ECC', repo[1]) - repo = '/'.join(repo) - return '%s#%d' % (repo, self.issue_number) - else: - return 'Unknown' - - def __eq__(self, other): - return (self.repo_id, self.issue_number) == (other.repo_id, other.issue_number) - - def __hash__(self): - return hash((self.repo_id, self.issue_number)) - - def any_cat(self, categories): - release_cat = self.is_release if 'releases' in categories else False - targets_cat = self.is_target if 'targets' in categories else False - return release_cat or targets_cat - -def fetch_issues(op, issues): - repos = set([repo for (repo, _) in issues]) - repos = {repo: [issue for (r, issue) in issues if r == repo] for repo in repos} - - for (repo, issues) in repos.items(): - conn = op.repository( - owner=REPOS[repo][0], - name=REPOS[repo][1], - __alias__='repo%d' % repo, - ) - - for issue in issues: - res = conn.issue_or_pull_request(number=issue, __alias__='issue%d' % issue) - for typ in [schema.Issue, schema.PullRequest]: - node = res.__as__(typ) - node.labels(first=50).nodes().name() - node.state() - node.milestone().title() - node.title() - node.url() - if typ == schema.PullRequest: - node.merged() - -def download_issues(endpoint, nodes): - issues = [(repo, issue) for (repo, issue) in nodes if repo in REPOS] - - ret = {} - - # Ensure that any graph nodes from ZenHub that are not in the repos we care about have - # default entries, to simplify subsequent graph manipulation code. - for (repo, issue) in [(repo, issue) for (repo, issue) in nodes if repo not in REPOS]: - ret[(repo, issue)] = GitHubIssue(repo, issue, None) - - def chunks(lst, n): - for i in range(0, len(lst), n): - yield lst[i:i + n] - - for issues in chunks(issues, 50): - op = Operation(schema.Query) - fetch_issues(op, issues) - - d = endpoint(op) - data = (op + d) - - for (repo, issue) in issues: - repo_data = data['repo%d' % repo] - issue_key = 'issue%d' % issue - # If GITHUB_TOKEN doesn't have permission to read from a particular private - # repository in REPOS, GitHub returns an empty repo_data section. - issue_data = repo_data[issue_key] if issue_key in repo_data else None - ret[(repo, issue)] = GitHubIssue(repo, issue, issue_data) - - return ret - -def fetch_workspace_graph(op, workspace_id, repos, cursor): - dependencies = op.workspace(id=workspace_id).issue_dependencies( - # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here, - # not the GitHub repo IDs (which the previous REST API used). - # repository_ids=repos, - first=100, - after=cursor, - ) - dependencies.nodes.id() - dependencies.nodes.blocked_issue.number() - dependencies.nodes.blocked_issue.repository.gh_id() - dependencies.nodes.blocking_issue.number() - dependencies.nodes.blocking_issue.repository.gh_id() - dependencies.page_info.has_next_page() - dependencies.page_info.end_cursor() - -def get_dependency_graph(endpoint, workspace_id, repos): - edges = [] - cursor = None - - while True: - op = Operation(zenhub_schema.Query) - fetch_workspace_graph(op, workspace_id, repos, cursor) - - d = endpoint(op) - data = (op + d) - - dependencies = data.workspace.issue_dependencies - edges += [ - ( - (node.blocking_issue.repository.gh_id, node.blocking_issue.number), - (node.blocked_issue.repository.gh_id, node.blocked_issue.number), - ) for node in dependencies.nodes - ] - - if dependencies.page_info.has_next_page: - cursor = dependencies.page_info.end_cursor - print('.', end='', flush=True) - else: - print() - break - - return nx.DiGraph(edges) - -def fetch_epics(op, workspace_id, repos, cursor): - epics = op.workspace(id=workspace_id).epics( - # TODO: This causes a 500 Internal Server Error. We need the ZenHub repo IDs here, - # not the GitHub repo IDs (which the previous REST API used). - # repository_ids=repos, - first=100, - after=cursor, - ) - epics.nodes.id() - epics.nodes.issue.number() - epics.nodes.issue.repository.gh_id() - epics.page_info.has_next_page() - epics.page_info.end_cursor() - -def get_epics(endpoint, workspace_id, repos): - epics = [] - cursor = None - - while True: - op = Operation(zenhub_schema.Query) - fetch_epics(op, workspace_id, repos, cursor) - - d = endpoint(op) - data = (op + d) - - epics_page = data.workspace.epics - epics += [ - (node.id, (node.issue.repository.gh_id, node.issue.number)) - for node in epics_page.nodes - ] - - if epics_page.page_info.has_next_page: - cursor = epics_page.page_info.end_cursor - print('.', end='', flush=True) - else: - print() - break - - return epics - -def fetch_epic_issues(op, workspace_id, epic_id, cursor): - epic = op.workspace(id=workspace_id).epics(ids=[epic_id]) - child_issues = epic.nodes.child_issues( - first=100, - after=cursor, - ) - child_issues.nodes.number() - child_issues.nodes.repository.gh_id() - child_issues.page_info.has_next_page() - child_issues.page_info.end_cursor() - -def get_epic_issues(endpoint, workspace_id, epic_id): - epic_issues = [] - cursor = None - - while True: - op = Operation(zenhub_schema.Query) - fetch_epic_issues(op, workspace_id, epic_id, cursor) - - d = endpoint(op) - data = (op + d) - - epic = data.workspace.epics.nodes[0] - epic_issues += [ - (node.repository.gh_id, node.number) - for node in epic.child_issues.nodes - ] - - if epic.child_issues.page_info.has_next_page: - cursor = epic.child_issues.page_info.end_cursor - print('.', end='', flush=True) - else: - print() - break - - return epic_issues - def main(): - gapi = HTTPEndpoint( - 'https://api.github.com/graphql', - {'Authorization': 'bearer %s' % GITHUB_TOKEN}, - ) - zapi = HTTPEndpoint( - 'https://api.zenhub.com/public/graphql', - {'Authorization': 'Bearer %s' % ZENHUB_TOKEN}, - ) + gapi = github.api(GITHUB_TOKEN) + zapi = zenhub.api(ZENHUB_TOKEN) # Build the full dependency graph from ZenHub's per-workspace API. print('Fetching graph') dg = nx.compose_all([ - get_dependency_graph(zapi, workspace_id, repos) + zenhub.get_dependency_graph(zapi, workspace_id, repos) for (workspace_id, repos) in WORKSPACES.items() if len(repos) > 0 ]) @@ -412,10 +71,10 @@ def main(): epics_issues = [] for (workspace_id, repos) in WORKSPACES.items(): if len(repos) > 0: - epics_issues += get_epics(zapi, workspace_id, repos) + epics_issues += zenhub.get_epics(zapi, workspace_id, repos) epics_issues = set(epics_issues) - epics_mapping = download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues]) + epics_mapping = github.download_issues(gapi, [gh_ref for (_, gh_ref) in epics_issues], REPOS) epics_mapping = {k: v for (k, v) in epics_mapping.items() if v.state != 'closed'} issues_by_epic = {} for (i, ((repo_id, epic_id), epic)) in enumerate(epics_mapping.items()): @@ -428,7 +87,7 @@ def main(): id for (id, gh_ref) in epics_issues if gh_ref == (repo_id, epic_id) ][0] - issues = set(get_epic_issues(zapi, workspace_id, epic_id)) + issues = set(zenhub.get_epic_issues(zapi, workspace_id, epic_id)) issues_by_epic[epic] = issues for i in issues: # zapi.dependencies only returns nodes that have some connection, @@ -448,7 +107,7 @@ def main(): dg = nx.subgraph(dg, terminate_at.union(*ancestors)) # Fetch the issues within the graph. - mapping = download_issues(gapi, dg.nodes) + mapping = github.download_issues(gapi, dg.nodes, REPOS) # Relabel the graph dg = nx.relabel_nodes(dg, mapping) From 3f769cb62d9274794b9869a39329ed934287110a Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 31 Oct 2024 17:39:46 +1300 Subject: [PATCH 2/6] Avoid breakage if a ZenHub workspace has no issue dependencies --- helpers/zenhub.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/helpers/zenhub.py b/helpers/zenhub.py index 00ee46d3a..93e28b487 100644 --- a/helpers/zenhub.py +++ b/helpers/zenhub.py @@ -52,18 +52,22 @@ def get_dependency_graph(endpoint, workspace_id, repos): d = endpoint(op) data = op + d - dependencies = data.workspace.issue_dependencies - edges += [ - ( - (node.blocking_issue.repository.gh_id, node.blocking_issue.number), - (node.blocked_issue.repository.gh_id, node.blocked_issue.number), - ) - for node in dependencies.nodes - ] - - if dependencies.page_info.has_next_page: - cursor = dependencies.page_info.end_cursor - print('.', end='', flush=True) + if hasattr(data.workspace, 'issue_dependencies'): + dependencies = data.workspace.issue_dependencies + edges += [ + ( + (node.blocking_issue.repository.gh_id, node.blocking_issue.number), + (node.blocked_issue.repository.gh_id, node.blocked_issue.number), + ) + for node in dependencies.nodes + ] + + if dependencies.page_info.has_next_page: + cursor = dependencies.page_info.end_cursor + print('.', end='', flush=True) + else: + print() + break else: print() break From a666c1ad762f07f618c67fa572e3daaa52b7725d Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 31 Oct 2024 17:41:44 +1300 Subject: [PATCH 3/6] Add a Zashi pipeline render This shows the progress of implementing tracked bugs and features, and deploying them to the mobile SDKs and Zashi apps. Closes zcash/developers#64. --- .github/workflows/gh-pages.yml | 6 + helpers/github.py | 94 ++++++++++++++ index.html | 1 + zashi-pipeline.py | 217 +++++++++++++++++++++++++++++++++ 4 files changed, 318 insertions(+) create mode 100644 zashi-pipeline.py diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 3fb670837..77c4b5506 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -148,6 +148,12 @@ jobs: DAG_VIEW: transparent-deprecation TERMINATE_AT: "zcash/zcash#4203" + - name: Render Zashi pipeline + run: python3 ./zashi-pipeline.py + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ZENHUB_TOKEN: ${{ secrets.ZENHUB_TOKEN }} + - name: Copy the index page run: cp ./index.html ./public diff --git a/helpers/github.py b/helpers/github.py index 1c701ba5b..28dfc3521 100644 --- a/helpers/github.py +++ b/helpers/github.py @@ -113,6 +113,7 @@ def __init__(self, repo_id, issue_number, data, REPOS): if data is not None: labels = [label['name'] for label in data['labels']['nodes']] self.title = data['title'] + self.labels = labels self.is_release = 'C-release' in labels self.is_target = 'C-target' in labels self.is_pr = 'merged' in data @@ -126,6 +127,7 @@ def __init__(self, repo_id, issue_number, data, REPOS): else: # If we can't fetch issue data, assume we don't care. self.title = '' + self.labels = [] self.url = None self.is_release = False self.is_target = False @@ -212,3 +214,95 @@ def chunks(lst, n): ret[(repo, issue)] = GitHubIssue(repo, issue, issue_data, REPOS) return ret + + +def fetch_issues_with_labels(op, labels, repos): + for (repo_id, (repo, issue_cursor, pr_cursor)) in repos: + conn = op.repository( + owner=repo[0], + name=repo[1], + __alias__='repo%d' % repo_id, + ) + + if issue_cursor != -1: + issues = conn.issues( + labels=labels, + first=50, + after=issue_cursor, + ) + issues.nodes.number() + issues.nodes.labels(first=50).nodes().name() + issues.nodes.state() + issues.nodes.milestone().title() + issues.nodes.title() + issues.nodes.url() + issues.page_info.has_next_page() + issues.page_info.end_cursor() + + if pr_cursor != -1: + prs = conn.pull_requests( + labels=labels, + first=50, + after=pr_cursor, + ) + prs.nodes.number() + prs.nodes.labels(first=50).nodes().name() + prs.nodes.state() + prs.nodes.milestone().title() + prs.nodes.title() + prs.nodes.url() + prs.nodes.merged() + prs.page_info.has_next_page() + prs.page_info.end_cursor() + + +def download_issues_with_labels(endpoint, labels, REPOS): + ret = {} + repos = {repo_id: (repo, None, None) for (repo_id, repo) in REPOS.items()} + + while True: + op = Operation(schema.Query) + fetch_issues_with_labels(op, labels, repos.items()) + + d = endpoint(op) + data = op + d + + done = [] + for (repo_id, (repo, _, _)) in repos.items(): + repo_data = data['repo%d' % repo_id] + + if hasattr(repo_data, 'issues'): + for issue in repo_data.issues.nodes: + ret[(repo_id, issue.number)] = GitHubIssue(repo_id, issue.number, issue, REPOS) + if repo_data.issues.page_info.has_next_page: + issue_cursor = repo_data.issues.page_info.end_cursor + else: + issue_cursor = -1 + else: + issue_cursor = -1 + + if hasattr(repo_data, 'pull_requests'): + for pr in repo_data.pull_requests.nodes: + ret[(repo_id, pr.number)] = GitHubIssue(repo_id, pr.number, pr, REPOS) + if repo_data.pull_requests.page_info.has_next_page: + pr_cursor = repo_data.pull_requests.page_info.end_cursor + else: + pr_cursor = -1 + else: + pr_cursor = -1 + + if issue_cursor == -1 and pr_cursor == -1: + done.append(repo_id) + else: + repos[repo_id] = (repo, issue_cursor, pr_cursor) + + for repo_id in done: + del repos[repo_id] + + if len(repos) > 0: + print('.', end='', flush=True) + else: + print() + break + + return ret diff --git a/index.html b/index.html index f3c42e178..415e8c7cd 100644 --- a/index.html +++ b/index.html @@ -17,5 +17,6 @@

Zcash Developers Hub

zcashd deprecation DAG

Sprout pool deprecation DAG

Transparent pool deprecation DAG

+

Zashi pipeline

diff --git a/zashi-pipeline.py b/zashi-pipeline.py new file mode 100644 index 000000000..f898a369b --- /dev/null +++ b/zashi-pipeline.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 + +import networkx as nx +from str2bool import str2bool as strtobool + +import os +import re +from textwrap import wrap +from urllib.parse import urlparse + +from helpers import github, zenhub + +GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN') +ZENHUB_TOKEN = os.environ.get('ZENHUB_TOKEN') + +# IDs of repos we look for releases in. +RUST = 85334928 +ANDROID_SDK = 151763639 +SWIFT_SDK = 185480114 +ZASHI_ANDROID = 390808594 +ZASHI_IOS = 387551125 + +REPOS = { + **github.CORE_REPOS, + **github.WALLET_REPOS, +} + + +class TrackedIssue: + def __init__(self, issue): + self.issue = issue + + +def main(): + gapi = github.api(GITHUB_TOKEN) + zapi = zenhub.api(ZENHUB_TOKEN) + + print('Fetching tracked issues') + tracked_issues = github.download_issues_with_labels(gapi, ['C-tracked-bug', 'C-tracked-feature'], REPOS) + + # The repos we care about are now: + # - Any repo containing a tracked issue. + # - The wallet repos where releases occur. + tracked_repos = set([repo_id for (repo_id, _) in tracked_issues]) + repos = { + **github.WALLET_REPOS, + } + for repo_id in tracked_repos: + repos[repo_id] = REPOS[repo_id] + workspaces = { + workspace_id: [repo_id for repo_id in repos if repo_id in repos] + for (workspace_id, repos) in zenhub.WORKSPACE_SETS.items() + } + + # Build the full dependency graph from ZenHub's per-workspace API. + print('Fetching graph') + dg = nx.compose_all([ + zenhub.get_dependency_graph(zapi, workspace_id, repos) + for (workspace_id, repos) in workspaces.items() + if len(repos) > 0 + ]) + + print('Rendering deployment pipeline') + + # Ensure that the tracked issues all exist in the graph. This is a no-op for + # issues that are already present. + start_at = set([issue for issue in tracked_issues]) + for i in start_at: + dg.add_node(i) + + # Replace the graph with the subgraph that only includes the tracked + # issues and their descendants. + descendants = [nx.descendants(dg, n) for n in start_at] + dg = nx.subgraph(dg, start_at.union(*descendants)) + + # Fetch the issues within the graph. + mapping = github.download_issues(gapi, dg.nodes, repos) + + # Relabel the graph + dg = nx.relabel_nodes(dg, mapping) + + # Filter out unknown issues + unknown = [n for n in dg if n.repo_id not in repos] + if len(unknown) > 0: + dg.remove_nodes_from(unknown) + + # Apply property annotations + for (source, sink) in dg.edges: + attrs = dg.edges[source, sink] + attrs['is_open'] = 0 if source.state == 'closed' else 1 + + # Render the HTML version! + html_header = ''' + + + Zashi Pipeline + + + + +

Zashi Pipeline

+

🐞 = bug, 💡 = feature, ✅ = implemented / released, 🛑 = unfinished, 📥 = unassigned / DAG needs updating

+ + + + + + + + + + + + + +''' + html_footer = ''' + +
TypeIssueRust crateAndroid SDKSwift SDKZashi AndroidZashi iOS
+ + +''' + with open('public/zashi-pipeline.html', 'w') as f: + f.write(html_header) + + for issue in tracked_issues.values(): + f.write('') + + if 'C-tracked-bug' in issue.labels: + f.write('🐞') + else: + f.write('💡') + + f.write('{} {}'.format( + '✅' if issue.state == 'closed' else '🛑', + issue.url, + issue.title, + )) + + children = nx.descendants(dg, issue) + def find_child_release(repo_id): + for child in children: + if child.repo_id == repo_id and 'C-release' in child.labels: + # Extract version number from title + if repo_id == RUST: + version = re.search(r'zcash_[^ ]+ \d+(\.\d+)+', child.title).group() + else: + version = re.search(r'\d+(\.\d+)+', child.title).group() + + f.write('{} {}'.format( + '✅' if child.state == 'closed' else '🛑', + child.url, + version, + )) + return + + # Release not found in this repo + f.write('📥') + + find_child_release(RUST) + find_child_release(ANDROID_SDK) + find_child_release(SWIFT_SDK) + find_child_release(ZASHI_ANDROID) + find_child_release(ZASHI_IOS) + + f.write('') + + f.write(html_footer) + + +if __name__ == '__main__': + if GITHUB_TOKEN and ZENHUB_TOKEN: + main() + else: + print('Please set the GITHUB_TOKEN and ZENHUB_TOKEN environment variables.') From ffe262ff3db2e0525bc2430608896b70260ee750 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 31 Oct 2024 16:32:32 -0600 Subject: [PATCH 4/6] Render each Zashi-related release that a given feature is blocking. --- zashi-pipeline.py | 77 +++++++++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/zashi-pipeline.py b/zashi-pipeline.py index f898a369b..002af2583 100644 --- a/zashi-pipeline.py +++ b/zashi-pipeline.py @@ -25,11 +25,37 @@ **github.WALLET_REPOS, } +RELEASE_MATRIX = { + RUST: [ANDROID_SDK, SWIFT_SDK], + ANDROID_SDK: [ZASHI_ANDROID], + SWIFT_SDK: [ZASHI_IOS], + ZASHI_ANDROID: [], + ZASHI_IOS: [] +} class TrackedIssue: def __init__(self, issue): self.issue = issue +def build_release_matrix_from(dg, issue, repo_id): + acc = [] + for child in dg.neighbors(issue): + if child.repo_id == repo_id and 'C-release' in child.labels: + child_releases = [] + for dep_repo in RELEASE_MATRIX.get(repo_id): + child_releases.extend(build_release_matrix_from(dg, child, dep_repo)) + + if len(child_releases) > 0: + for rec in child_releases: + rec[repo_id] = child + else: + child_releases = [{repo_id: child}] + + acc.extend(child_releases) + else: + acc.extend(build_release_matrix_from(dg, child, repo_id)) + + return acc def main(): gapi = github.api(GITHUB_TOKEN) @@ -166,23 +192,27 @@ def main(): f.write(html_header) for issue in tracked_issues.values(): - f.write('') - - if 'C-tracked-bug' in issue.labels: - f.write('🐞') - else: - f.write('💡') - - f.write('{} {}'.format( - '✅' if issue.state == 'closed' else '🛑', - issue.url, - issue.title, - )) - - children = nx.descendants(dg, issue) - def find_child_release(repo_id): - for child in children: - if child.repo_id == repo_id and 'C-release' in child.labels: + rows = build_release_matrix_from(dg, issue, RUST); + for row in rows: + f.write('') + + if 'C-tracked-bug' in issue.labels: + f.write('🐞') + else: + f.write('💡') + + f.write('{} {}'.format( + '✅' if issue.state == 'closed' else '🛑', + issue.url, + issue.title, + )) + + for repo_id in [RUST, ANDROID_SDK, SWIFT_SDK, ZASHI_ANDROID, ZASHI_IOS]: + child = row.get(repo_id) + if child is None: + # Release not found in this repo + f.write('📥') + else: # Extract version number from title if repo_id == RUST: version = re.search(r'zcash_[^ ]+ \d+(\.\d+)+', child.title).group() @@ -194,18 +224,7 @@ def find_child_release(repo_id): child.url, version, )) - return - - # Release not found in this repo - f.write('📥') - - find_child_release(RUST) - find_child_release(ANDROID_SDK) - find_child_release(SWIFT_SDK) - find_child_release(ZASHI_ANDROID) - find_child_release(ZASHI_IOS) - - f.write('') + f.write('') f.write(html_footer) From fc99f7418ed9abb36b1977d241a69b796994e3ee Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 1 Nov 2024 18:44:13 +1300 Subject: [PATCH 5/6] Merge branches of the same release tree into the same row --- zashi-pipeline.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/zashi-pipeline.py b/zashi-pipeline.py index 002af2583..db781f161 100644 --- a/zashi-pipeline.py +++ b/zashi-pipeline.py @@ -3,6 +3,7 @@ import networkx as nx from str2bool import str2bool as strtobool +import itertools import os import re from textwrap import wrap @@ -41,9 +42,17 @@ def build_release_matrix_from(dg, issue, repo_id): acc = [] for child in dg.neighbors(issue): if child.repo_id == repo_id and 'C-release' in child.labels: - child_releases = [] - for dep_repo in RELEASE_MATRIX.get(repo_id): - child_releases.extend(build_release_matrix_from(dg, child, dep_repo)) + # Fetch the rows that each child's downstreams need rendered. + child_deps = [ + build_release_matrix_from(dg, child, dep_repo) + for dep_repo in RELEASE_MATRIX.get(repo_id) + ] + + # Merge the rows from each downstream repo together. + child_releases = [ + {k: v for d in prod for k, v in d.items()} + for prod in itertools.product(*child_deps) + ] if len(child_releases) > 0: for rec in child_releases: From 29c7e5ae94d6901b4ba1bc67dc3fac721f4a9176 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 1 Nov 2024 18:51:37 +1300 Subject: [PATCH 6/6] Only render a tracked issue once for its group --- zashi-pipeline.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/zashi-pipeline.py b/zashi-pipeline.py index db781f161..b6e1e4825 100644 --- a/zashi-pipeline.py +++ b/zashi-pipeline.py @@ -202,19 +202,24 @@ def main(): for issue in tracked_issues.values(): rows = build_release_matrix_from(dg, issue, RUST); - for row in rows: + for i, row in enumerate(rows): f.write('') - if 'C-tracked-bug' in issue.labels: - f.write('🐞') - else: - f.write('💡') - - f.write('{} {}'.format( - '✅' if issue.state == 'closed' else '🛑', - issue.url, - issue.title, - )) + if i == 0: + rowspan = '' + if len(rows) > 1: + rowspan = ' rowspan="{}"'.format(len(rows)) + + f.write('{}'.format( + rowspan, + '🐞' if 'C-tracked-bug' in issue.labels else '💡', + )) + f.write('{} {}'.format( + rowspan, + '✅' if issue.state == 'closed' else '🛑', + issue.url, + issue.title, + )) for repo_id in [RUST, ANDROID_SDK, SWIFT_SDK, ZASHI_ANDROID, ZASHI_IOS]: child = row.get(repo_id)