diff --git a/ipstools/IPApproX_common.py b/ipstools/IPApproX_common.py index d3cbc08..e7004d4 100644 --- a/ipstools/IPApproX_common.py +++ b/ipstools/IPApproX_common.py @@ -99,6 +99,10 @@ def load_ips_list(filename, skip_commit=False): group = ips_list[i]['group'] except KeyError: group = None + try: + access_token = ips_list[i]['access_token'] + except KeyError: + access_token = None try: path = ips_list[i]['path'] except KeyError: @@ -108,7 +112,7 @@ def load_ips_list(filename, skip_commit=False): alternatives = list(set.union(set(ips_list[i]['alternatives']), set([name]))) except KeyError: alternatives = None - ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'path': path, 'domain': domain, 'alternatives': alternatives }) + ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'access_token': access_token, 'path': path, 'domain': domain, 'alternatives': alternatives }) return ips def store_ips_list(filename, ips): @@ -122,16 +126,21 @@ def store_ips_list(filename, ips): f.write(IPS_LIST_PREAMBLE) f.write(yaml.dump(ips_list)) -def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', verbose=False): +def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', access_token=None, verbose=False): with open(os.devnull, "w") as devnull: rawcontent_failed = False ips_list_yml = " " - if "github.com" in server: + if "gitlab." in server or "github.com" in server: if "tags/" in commit: commit = commit[5:] + if "gitlab." in server: + host = re.sub(r'^.*(@|//)([^:/]+).*', r'\2', server) + url = "https://%s/api/v4/projects/%s%%2F%s/repository/files/ips_list.yml/raw?ref=%s&private_token=%s" % (host, group, name, commit, access_token) + else: # assume github + url = "https://raw.githubusercontent.com/%s/%s/%s/ips_list.yml" % (group, name, commit) if verbose: - print(" Fetching ips_list.yml from https://raw.githubusercontent.com/%s/%s/%s/ips_list.yml" % (group, name, commit)) - cmd = "curl https://raw.githubusercontent.com/%s/%s/%s/ips_list.yml" % (group, name, commit) + print(" Fetching ips_list.yml from %s" % (url)) + cmd = "curl %s" % (url) try: curl = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=devnull) cmd = "cat" @@ -140,9 +149,12 @@ def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpi except subprocess.CalledProcessError: rawcontent_failed = True ips_list_yml = ips_list_yml.decode(sys.stdout.encoding) + if ("gitlab." in server and ips_list_yml[12] in "45"): + rawcontent_failed = True; if ips_list_yml[:3] == "404": ips_list_yml = "" - if rawcontent_failed or "github.com" not in server: + # Fall back to git archive, which will not work for hashes that don't point to any branch head + if rawcontent_failed or ("gitlab." not in server and "github.com" not in server): if verbose: print(" Fetching ips_list.yml from %s:%s/%s @ %s" % (server, group, name, commit)) cmd = "git archive --remote=%s:%s/%s %s ips_list.yml" % (server, group, name, commit) @@ -157,8 +169,8 @@ def get_ips_list_yml(server="git@github.com", group='pulp-platform', name='pulpi ips_list_yml = ips_list_yml.decode(sys.stdout.encoding) return ips_list_yml -def load_ips_list_from_server(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', verbose=False, skip_commit=False): - ips_list_yml = get_ips_list_yml(server, group, name, commit, verbose=verbose) +def load_ips_list_from_server(server="git@github.com", group='pulp-platform', name='pulpissimo.git', commit='master', access_token=None, verbose=False, skip_commit=False): + ips_list_yml = get_ips_list_yml(server, group, name, commit, access_token, verbose=verbose) if ips_list_yml is None: print("No ips_list.yml gathered for %s" % name) return [] @@ -183,6 +195,10 @@ def load_ips_list_from_server(server="git@github.com", group='pulp-platform', na group = ips_list[i]['group'] except KeyError: group = None + try: + access_token = ips_list[i]['access_token'] + except KeyError: + access_token = None try: path = ips_list[i]['path'] except KeyError: @@ -192,7 +208,7 @@ def load_ips_list_from_server(server="git@github.com", group='pulp-platform', na alternatives = list(set.union(set(ips_list[i]['alternatives']), set([name]))) except KeyError: alternatives = None - ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'path': path, 'domain': domain, 'alternatives': alternatives }) + ips.append({'name': name, 'commit': commit, 'server': server, 'group': group, 'access_token': access_token, 'path': path, 'domain': domain, 'alternatives': alternatives }) except AttributeError: # here it fails silently (by design). it means that at the same time # 1. the ip's version is a commit hash, not a branch or tag diff --git a/ipstools/IPDatabase.py b/ipstools/IPDatabase.py index 64687b7..401dc9f 100644 --- a/ipstools/IPDatabase.py +++ b/ipstools/IPDatabase.py @@ -600,7 +600,7 @@ def update_ips(self, origin='origin', force_downgrade=True): # make sure we have the correct branch/tag for the pull date_current = int(execute_out("%s show -s --format=%%ct HEAD" % git).rstrip()) - lines = execute_out("%s show -s --format=%%ct %s" % (git, ip['commit'])).splitlines() + lines = execute_out("%s show -s --format=%%ct %s --" % (git, ip['commit'])).splitlines() date_specified = int(lines[-1]) if (date_current > date_specified) and not force_downgrade: diff --git a/ipstools/IPTreeNode.py b/ipstools/IPTreeNode.py index 98169a0..f584b6a 100644 --- a/ipstools/IPTreeNode.py +++ b/ipstools/IPTreeNode.py @@ -71,7 +71,7 @@ def __init__(self, commit = node['commit'] else: commit = default_commit - ips = load_ips_list_from_server(server, group, node['name'], commit, verbose=verbose) + ips = load_ips_list_from_server(server, group, node['name'], commit, access_token=node['access_token'], verbose=verbose) father_of_children = { 'server' : server, 'group' : group, diff --git a/ipstools/SubIPConfig.py b/ipstools/SubIPConfig.py index 4c2e638..62a402f 100644 --- a/ipstools/SubIPConfig.py +++ b/ipstools/SubIPConfig.py @@ -161,7 +161,7 @@ def export_make(self, abs_path, more_opts, target_tech=None, local=False, simula vhdl_files = "" vlog_files = "" for f in files: - if f[0] == '/': + if f[0] == '/' or f[0] == '$': if not is_vhdl(f): vlog_files += "\\\n\t%s" % (f) else: