From e69ade7d80276ba806ca99a4fd698a30efcebfc0 Mon Sep 17 00:00:00 2001 From: Stefan Tauner Date: Wed, 28 Jul 2021 02:47:42 +0200 Subject: [PATCH 1/4] Treat src files starting with $ like absolute paths --- ipstools/SubIPConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipstools/SubIPConfig.py b/ipstools/SubIPConfig.py index 4c2e638..62a402f 100644 --- a/ipstools/SubIPConfig.py +++ b/ipstools/SubIPConfig.py @@ -161,7 +161,7 @@ def export_make(self, abs_path, more_opts, target_tech=None, local=False, simula vhdl_files = "" vlog_files = "" for f in files: - if f[0] == '/': + if f[0] == '/' or f[0] == '$': if not is_vhdl(f): vlog_files += "\\\n\t%s" % (f) else: From b5bdcbcb8ae5828d03f40409eead5be07aaf2ef2 Mon Sep 17 00:00:00 2001 From: Stefan Tauner Date: Tue, 8 Nov 2022 01:40:45 +0100 Subject: [PATCH 2/4] Add support for fetching ips_lists.yml from gitlab --- ipstools/IPApproX_common.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ipstools/IPApproX_common.py b/ipstools/IPApproX_common.py index d3cbc08..f217093 100644 --- a/ipstools/IPApproX_common.py +++ b/ipstools/IPApproX_common.py @@ -122,16 +122,21 @@ def store_ips_list(filename, ips): f.write(IPS_LIST_PREAMBLE) f.write(yaml.dump(ips_list)) -def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', verbose=False): +def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', access_token=None, verbose=False): with open(os.devnull, "w") as devnull: rawcontent_failed = False ips_list_yml = " " - if "github.com" in server: + if "gitlab." in server or "github.com" in server: if "tags/" in commit: commit = commit[5:] + if "gitlab." in server: + host = re.sub(r'^.*(@|//)([^:/]+).*', r'\2', server) + url = "https://%s/api/v4/projects/%s%%2F%s/repository/files/ips_list.yml/raw?ref=%s&private_token=%s" % (host, group, name, commit, access_token) + else: # assume github + url = "https://raw.githubusercontent.com/%s/%s/%s/ips_list.yml" % (group, name, commit) if verbose: - print(" Fetching ips_list.yml from https://raw.githubusercontent.com/%s/%s/%s/ips_list.yml" % (group, name, commit)) - cmd = "curl https://raw.githubusercontent.com/%s/%s/%s/ips_list.yml" % (group, name, commit) + print(" Fetching ips_list.yml from %s" % (url)) + cmd = "curl %s" % (url) try: curl = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=devnull) cmd = "cat" @@ -140,9 +145,12 @@ def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpi except subprocess.CalledProcessError: rawcontent_failed = True ips_list_yml = ips_list_yml.decode(sys.stdout.encoding) + if ("gitlab." in server and ips_list_yml[12] in "45"): + rawcontent_failed = True; if ips_list_yml[:3] == "404": ips_list_yml = "" - if rawcontent_failed or "github.com" not in server: + # Fall back to git archive, which will not work for hashes that don't point to any branch head + if rawcontent_failed or ("gitlab." not in server and "github.com" not in server): if verbose: print(" Fetching ips_list.yml from %s:%s/%s @ %s" % (server, group, name, commit)) cmd = "git archive --remote=%s:%s/%s %s ips_list.yml" % (server, group, name, commit) From 77cf2e7d34476b06b61fff84475da0faa43dc556 Mon Sep 17 00:00:00 2001 From: Stefan Tauner Date: Tue, 8 Nov 2022 05:03:27 +0100 Subject: [PATCH 3/4] Add rudimentary support for access_token fields in ips lists --- ipstools/IPApproX_common.py | 16 ++++++++++++---- ipstools/IPTreeNode.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ipstools/IPApproX_common.py b/ipstools/IPApproX_common.py index f217093..e7004d4 100644 --- a/ipstools/IPApproX_common.py +++ b/ipstools/IPApproX_common.py @@ -99,6 +99,10 @@ def load_ips_list(filename, skip_commit=False): group = ips_list[i]['group'] except KeyError: group = None + try: + access_token = ips_list[i]['access_token'] + except KeyError: + access_token = None try: path = ips_list[i]['path'] except KeyError: @@ -108,7 +112,7 @@ def load_ips_list(filename, skip_commit=False): alternatives = list(set.union(set(ips_list[i]['alternatives']), set([name]))) except KeyError: alternatives = None - ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'path': path, 'domain': domain, 'alternatives': alternatives }) + ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'access_token': access_token, 'path': path, 'domain': domain, 'alternatives': alternatives }) return ips def store_ips_list(filename, ips): @@ -165,8 +169,8 @@ def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpi ips_list_yml = ips_list_yml.decode(sys.stdout.encoding) return ips_list_yml -def load_ips_list_from_server(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', verbose=False, skip_commit=False): - ips_list_yml = get_ips_list_yml(server, group, name, commit, verbose=verbose) +def load_ips_list_from_server(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', access_token=None, verbose=False, skip_commit=False): + ips_list_yml = get_ips_list_yml(server, group, name, commit, access_token, verbose=verbose) if ips_list_yml is None: print("No ips_list.yml gathered for %s" % name) return [] @@ -191,6 +195,10 @@ def load_ips_list_from_server(server="git@github.com", group='pulp-platform', na group = ips_list[i]['group'] except KeyError: group = None + try: + access_token = ips_list[i]['access_token'] + except KeyError: + access_token = None try: path = ips_list[i]['path'] except KeyError: @@ -200,7 +208,7 @@ def load_ips_list_from_server(server="git@github.com", group='pulp-platform', na alternatives = list(set.union(set(ips_list[i]['alternatives']), set([name]))) except KeyError: alternatives = None - ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'path': path, 'domain': domain, 'alternatives': alternatives }) + ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'access_token': access_token, 'path': path, 'domain': domain, 'alternatives': alternatives }) except AttributeError: # here it fails silently (by design). it means that at the same time # 1. the ip's version is a commit hash, not a branch or tag diff --git a/ipstools/IPTreeNode.py b/ipstools/IPTreeNode.py index 98169a0..f584b6a 100644 --- a/ipstools/IPTreeNode.py +++ b/ipstools/IPTreeNode.py @@ -71,7 +71,7 @@ def __init__(self, commit = node['commit'] else: commit = default_commit - ips = load_ips_list_from_server(server, group, node['name'], commit, verbose=verbose) + ips = load_ips_list_from_server(server, group, node['name'], commit, access_token=node['access_token'], verbose=verbose) father_of_children = { 'server' : server, 'group' : group, From aeec5520d7474194b2da9f6d0b10a5a0c42cb316 Mon Sep 17 00:00:00 2001 From: Stefan Tauner Date: Tue, 20 Feb 2024 18:19:08 +0100 Subject: [PATCH 4/4] Make git command to determine current branch's date more robust --- ipstools/IPDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipstools/IPDatabase.py b/ipstools/IPDatabase.py index 64687b7..401dc9f 100644 --- a/ipstools/IPDatabase.py +++ b/ipstools/IPDatabase.py @@ -600,7 +600,7 @@ def update_ips(self, origin='origin', force_downgrade=True): # make sure we have the correct branch/tag for the pull date_current = int(execute_out("%s show -s --format=%%ct HEAD" % git).rstrip()) - lines = execute_out("%s show -s --format=%%ct %s" % (git, ip['commit'])).splitlines() + lines = execute_out("%s show -s --format=%%ct %s --" % (git, ip['commit'])).splitlines() date_specified = int(lines[-1]) if (date_current > date_specified) and not force_downgrade: