From c52aa462cb290083a7423e0bccb8289a21d00a61 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Wed, 6 Jan 2021 13:08:25 +0100 Subject: [PATCH 01/48] use osx sysroot --- cget/cmake/autotools.cmake | 12 +++++++++--- cget/cmake/boost.cmake | 12 +++++++++--- cget/cmake/make.cmake | 12 +++++++++--- cget/cmake/meson.cmake | 12 +++++++++--- tools/preamble.cmake | 12 +++++++++--- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index 5dc9417..3701785 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -103,6 +103,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -111,9 +120,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 3f8a49a..87127ec 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -96,6 +96,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -104,9 +113,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/cget/cmake/make.cmake b/cget/cmake/make.cmake index 40353aa..7d5f495 100644 --- a/cget/cmake/make.cmake +++ b/cget/cmake/make.cmake @@ -101,6 +101,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -109,9 +118,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/cget/cmake/meson.cmake b/cget/cmake/meson.cmake index f78c1b7..73e4601 100644 --- a/cget/cmake/meson.cmake +++ b/cget/cmake/meson.cmake @@ -130,6 +130,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -138,9 +147,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/tools/preamble.cmake b/tools/preamble.cmake index d95ca55..ae1fed6 100644 --- a/tools/preamble.cmake +++ b/tools/preamble.cmake @@ -86,6 +86,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -94,9 +103,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) From 8ab8918e2365b5e5aaa9495ef1c8eebb8d84f2e5 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Wed, 6 Jan 2021 14:08:29 +0100 Subject: [PATCH 02/48] use requests instead of urlopen to avoid 403 --- cget/util.py | 40 ++++++++++++++++++---------------------- requirements.txt | 1 + 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..f7e2457 100644 --- a/cget/util.py +++ b/cget/util.py @@ -16,7 +16,7 @@ else: import subprocess -from six.moves.urllib import request +import requests def to_bool(value): x = str(value).lower() @@ -205,31 +205,27 @@ def symlink_to(src, dst_dir): os.symlink(src, target) return target -class CGetURLOpener(request.FancyURLopener): - def http_error_default(self, url, fp, errcode, errmsg, headers): - if errcode >= 400: - raise BuildError("Download failed with error {0} for: {1}".format(errcode, url)) - return request.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) - def download_to(url, download_dir, insecure=False): name = url.split('/')[-1] - file = os.path.join(download_dir, name) + file_name = os.path.join(download_dir, name) click.echo("Downloading {0}".format(url)) - bar_len = 1000 - with click.progressbar(length=bar_len, width=70) as bar: - def hook(count, block_size, total_size): - percent = int(count*block_size*bar_len/total_size) - if percent > 0 and percent < bar_len: - # Hack because we can't set the position - bar.pos = percent - bar.update(0) - context = None - if insecure: context = ssl._create_unverified_context() - CGetURLOpener(context=context).retrieve(url, filename=file, reporthook=hook, data=None) - bar.update(bar_len) - if not os.path.exists(file): + with open(file_name, "wb") as f: + response = requests.get(url, stream=True) + total_length = response.headers.get('content-length') + if total_length is None: # no content length header + f.write(response.content) + else: + dl = 0 + total_length = int(total_length) + for data in response.iter_content(chunk_size=4096): + dl += len(data) + f.write(data) + done = int(50 * dl / total_length) + sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) ) + sys.stdout.flush() + if not os.path.exists(file_name): raise BuildError("Download failed for: {0}".format(url)) - return file + return file_name def transfer_to(f, dst, copy=False): if USE_SYMLINKS and not copy: return symlink_to(f, dst) diff --git a/requirements.txt b/requirements.txt index 1df73e3..2e2a871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ click>=6.6 # PyYAML six>=1.10 +requests From 7c11f74b176d9b05556db1193e3e963f607c34f4 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Wed, 6 Jan 2021 14:08:29 +0100 Subject: [PATCH 03/48] use requests instead of urlopen to avoid 403 --- cget/util.py | 40 ++++++++++++++++++---------------------- requirements.txt | 1 + 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..f7e2457 100644 --- a/cget/util.py +++ b/cget/util.py @@ -16,7 +16,7 @@ else: import subprocess -from six.moves.urllib import request +import requests def to_bool(value): x = str(value).lower() @@ -205,31 +205,27 @@ def symlink_to(src, dst_dir): os.symlink(src, target) return target -class CGetURLOpener(request.FancyURLopener): - def http_error_default(self, url, fp, errcode, errmsg, headers): - if errcode >= 400: - raise BuildError("Download failed with error {0} for: {1}".format(errcode, url)) - return request.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) - def download_to(url, download_dir, insecure=False): name = url.split('/')[-1] - file = os.path.join(download_dir, name) + file_name = os.path.join(download_dir, name) click.echo("Downloading {0}".format(url)) - bar_len = 1000 - with click.progressbar(length=bar_len, width=70) as bar: - def hook(count, block_size, total_size): - percent = int(count*block_size*bar_len/total_size) - if percent > 0 and percent < bar_len: - # Hack because we can't set the position - bar.pos = percent - bar.update(0) - context = None - if insecure: context = ssl._create_unverified_context() - CGetURLOpener(context=context).retrieve(url, filename=file, reporthook=hook, data=None) - bar.update(bar_len) - if not os.path.exists(file): + with open(file_name, "wb") as f: + response = requests.get(url, stream=True) + total_length = response.headers.get('content-length') + if total_length is None: # no content length header + f.write(response.content) + else: + dl = 0 + total_length = int(total_length) + for data in response.iter_content(chunk_size=4096): + dl += len(data) + f.write(data) + done = int(50 * dl / total_length) + sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) ) + sys.stdout.flush() + if not os.path.exists(file_name): raise BuildError("Download failed for: {0}".format(url)) - return file + return file_name def transfer_to(f, dst, copy=False): if USE_SYMLINKS and not copy: return symlink_to(f, dst) diff --git a/requirements.txt b/requirements.txt index 1df73e3..2e2a871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ click>=6.6 # PyYAML six>=1.10 +requests From 0051c345a7c2e10848842704d62b64cdfb4cc670 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 7 Jan 2021 07:04:34 +0100 Subject: [PATCH 04/48] drop dead import --- cget/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cget/util.py b/cget/util.py index f7e2457..0812320 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib if sys.version_info[0] < 3: try: From 70e05749392a04658f8c7d1f30740dad5b110731 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 7 Jan 2021 07:30:33 +0100 Subject: [PATCH 05/48] drop ssl import --- cget/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cget/util.py b/cget/util.py index f7e2457..0812320 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib if sys.version_info[0] < 3: try: From fa9af070cffcc5ec29fe358164ecd9eb38e2f1c6 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 7 Jan 2021 07:30:42 +0100 Subject: [PATCH 06/48] use click progressbar --- cget/util.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cget/util.py b/cget/util.py index 0812320..7acf966 100644 --- a/cget/util.py +++ b/cget/util.py @@ -215,14 +215,13 @@ def download_to(url, download_dir, insecure=False): if total_length is None: # no content length header f.write(response.content) else: - dl = 0 total_length = int(total_length) - for data in response.iter_content(chunk_size=4096): - dl += len(data) - f.write(data) - done = int(50 * dl / total_length) - sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) ) - sys.stdout.flush() + with click.progressbar(length=total_length, width=70) as bar: + for data in response.iter_content(chunk_size=4096): + f.write(data) + bar.pos += len(data) + bar.update(0) + bar.update(total_length) if not os.path.exists(file_name): raise BuildError("Download failed for: {0}".format(url)) return file_name From 0a6907906613a8e76ead8d243e5324a4f01414a4 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 7 Jan 2021 10:53:54 +0100 Subject: [PATCH 07/48] handle download error --- cget/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cget/util.py b/cget/util.py index 7acf966..7747fa1 100644 --- a/cget/util.py +++ b/cget/util.py @@ -211,6 +211,7 @@ def download_to(url, download_dir, insecure=False): click.echo("Downloading {0}".format(url)) with open(file_name, "wb") as f: response = requests.get(url, stream=True) + response.raise_for_status() total_length = response.headers.get('content-length') if total_length is None: # no content length header f.write(response.content) From 21cd83ab7f78868105dc3dc5b68b916ac0ddbae7 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 8 Jan 2021 15:09:49 +0100 Subject: [PATCH 08/48] dont use global include dir --- cget/prefix.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index ad8e658..6444c04 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -124,13 +124,6 @@ def generate_cmake_toolchain(self, toolchain=None, cc=None, cxx=None, cflags=Non append_ = cmake_append yield set_('CGET_PREFIX', self.prefix) yield set_('CMAKE_PREFIX_PATH', self.prefix) - yield if_('${CMAKE_VERSION} VERSION_LESS "3.6.0"', - ['include_directories(SYSTEM ${CGET_PREFIX}/include)'], - else_( - set_('CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include'), - set_('CMAKE_C_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include') - ) - ) if toolchain: yield ['include({})'.format(util.quote(os.path.abspath(toolchain)))] yield if_('CMAKE_CROSSCOMPILING', append_('CMAKE_FIND_ROOT_PATH', self.prefix) From 9e9c9196a555c2b37bb81e7d1f2dd5a787d2ff2d Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 8 Jan 2021 16:55:29 +0100 Subject: [PATCH 09/48] implement simple package hashing --- cget/cli.py | 2 ++ cget/package.py | 8 +++++++- cget/prefix.py | 11 ++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..f321492 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -100,6 +100,8 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato else: file = 'requirements.txt' pbs = [PackageBuild(pkg, cmake=cmake, variant=variant) for pkg in pkgs] for pbu in util.flat([prefix.from_file(file), pbs]): + hash = prefix.hash_pkg(pbu) + print("package '%s' hash %s" % (prefix.parse_pkg_src(pbu).name, hash)) pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) diff --git a/cget/package.py b/cget/package.py index 8a93c79..649829e 100644 --- a/cget/package.py +++ b/cget/package.py @@ -1,4 +1,4 @@ -import base64, copy, argparse, six +import base64, copy, argparse, six, dirhash, hashlib def encode_url(url): x = six.b(url[url.find('://')+3:]) @@ -31,6 +31,12 @@ def get_src_dir(self): return self.url[7:] # Remove "file://" raise TypeError() + def calc_hash(self): + if self.recipe: + return dirhash.dirhash(self.recipe, "sha1") + elif self.url: + return hashlib.sha1(self.url.encode("utf-8")).hexdigest() + raise Exception("no url or recipe: %s" % self.__dict__) def fname_to_pkg(fname): if fname.startswith('_url_'): return PackageSource(name=decode_url(fname), fname=fname) diff --git a/cget/prefix.py b/cget/prefix.py index ad8e658..37de659 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -1,4 +1,4 @@ -import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools +import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools, hashlib from cget.builder import Builder from cget.package import fname_to_pkg @@ -223,6 +223,15 @@ def parse_src_github(self, name, url): if name is None: name = p return PackageSource(name=name, url=url) + def hash_pkg(self, pkg): + pkg_src = self.parse_pkg_src(pkg) + result = pkg_src.calc_hash() + pkg_build = self.parse_pkg_build(pkg) + if pkg_build.requirements: + for dependency in self.from_file(pkg_build.requirements): + result = hashlib.sha1((result + self.hash_pkg(dependency)).encode("utf-8")).hexdigest() + return result + @returns(PackageSource) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_src(self, pkg, start=None, no_recipe=False): From 37a64d2027ee5700b7145ba4ccbda7ab7d136587 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 8 Jan 2021 16:55:29 +0100 Subject: [PATCH 10/48] implement simple package hashing --- cget/cli.py | 2 ++ cget/package.py | 8 +++++++- cget/prefix.py | 11 ++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..f321492 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -100,6 +100,8 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato else: file = 'requirements.txt' pbs = [PackageBuild(pkg, cmake=cmake, variant=variant) for pkg in pkgs] for pbu in util.flat([prefix.from_file(file), pbs]): + hash = prefix.hash_pkg(pbu) + print("package '%s' hash %s" % (prefix.parse_pkg_src(pbu).name, hash)) pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) diff --git a/cget/package.py b/cget/package.py index 8a93c79..649829e 100644 --- a/cget/package.py +++ b/cget/package.py @@ -1,4 +1,4 @@ -import base64, copy, argparse, six +import base64, copy, argparse, six, dirhash, hashlib def encode_url(url): x = six.b(url[url.find('://')+3:]) @@ -31,6 +31,12 @@ def get_src_dir(self): return self.url[7:] # Remove "file://" raise TypeError() + def calc_hash(self): + if self.recipe: + return dirhash.dirhash(self.recipe, "sha1") + elif self.url: + return hashlib.sha1(self.url.encode("utf-8")).hexdigest() + raise Exception("no url or recipe: %s" % self.__dict__) def fname_to_pkg(fname): if fname.startswith('_url_'): return PackageSource(name=decode_url(fname), fname=fname) diff --git a/cget/prefix.py b/cget/prefix.py index ad8e658..37de659 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -1,4 +1,4 @@ -import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools +import os, shutil, shlex, six, inspect, click, contextlib, uuid, sys, functools, hashlib from cget.builder import Builder from cget.package import fname_to_pkg @@ -223,6 +223,15 @@ def parse_src_github(self, name, url): if name is None: name = p return PackageSource(name=name, url=url) + def hash_pkg(self, pkg): + pkg_src = self.parse_pkg_src(pkg) + result = pkg_src.calc_hash() + pkg_build = self.parse_pkg_build(pkg) + if pkg_build.requirements: + for dependency in self.from_file(pkg_build.requirements): + result = hashlib.sha1((result + self.hash_pkg(dependency)).encode("utf-8")).hexdigest() + return result + @returns(PackageSource) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_src(self, pkg, start=None, no_recipe=False): From c00433960336f9506f6ec4a2b58aac613b1c56c6 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 8 Jan 2021 17:24:38 +0100 Subject: [PATCH 11/48] zip builds to cache --- cget/prefix.py | 1 + cget/util.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/cget/prefix.py b/cget/prefix.py index 37de659..8d50e86 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -338,6 +338,7 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, if test or test_all: builder.test(variant=pb.variant) # Install builder.build(target='install', variant=pb.variant) + util.zip_dir_to_cache("builds/%s" % pb.to_name(), self.hash_pkg(pb), install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..6ef75aa 100644 --- a/cget/util.py +++ b/cget/util.py @@ -87,6 +87,25 @@ def mkfile(d, file, content, always_write=True): write_to(p, content) return p +def zipdir(src_dir, tgt_file): + print("zipping '%s' to '%s" % (src_dir, tgt_file)) + zipf = zipfile.ZipFile(tgt_file, 'w', zipfile.ZIP_DEFLATED) + for root, dirs, files in os.walk(src_dir): + for file in files: + zipf.write( + os.path.join(root, file), + os.path.relpath( + os.path.join(root, file), + os.path.join(src_dir, '..') + ) + ) + zipf.close() + +def zip_dir_to_cache(prefix, key, src_dir): + out_dir = get_cache_path(prefix) + mkdir(out_dir) + zipdir(src_dir, os.path.join(out_dir, key + ".zip")) + def ls(p, predicate=lambda x:True): if os.path.exists(p): return (d for d in os.listdir(p) if predicate(os.path.join(p, d))) From 8dcd3ab22a18bc867bbcaecc819a1cbb03393044 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 09:50:52 +0100 Subject: [PATCH 12/48] Revert "dont use global include dir" This reverts commit 21cd83ab7f78868105dc3dc5b68b916ac0ddbae7. --- cget/prefix.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cget/prefix.py b/cget/prefix.py index 6444c04..ad8e658 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -124,6 +124,13 @@ def generate_cmake_toolchain(self, toolchain=None, cc=None, cxx=None, cflags=Non append_ = cmake_append yield set_('CGET_PREFIX', self.prefix) yield set_('CMAKE_PREFIX_PATH', self.prefix) + yield if_('${CMAKE_VERSION} VERSION_LESS "3.6.0"', + ['include_directories(SYSTEM ${CGET_PREFIX}/include)'], + else_( + set_('CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include'), + set_('CMAKE_C_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include') + ) + ) if toolchain: yield ['include({})'.format(util.quote(os.path.abspath(toolchain)))] yield if_('CMAKE_CROSSCOMPILING', append_('CMAKE_FIND_ROOT_PATH', self.prefix) From 4048bcf10e58b70a22479e9cfc7fe8bc75bccf67 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 09:56:58 +0100 Subject: [PATCH 13/48] make global include dir optional --- cget/cli.py | 7 +++++-- cget/prefix.py | 26 +++++++++++++++++++------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..0c3a0b2 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -56,7 +56,8 @@ def w(obj, prefix, verbose, build_path, *args, **kwargs): @click.option('-D', '--define', multiple=True, help="Extra configuration variables to pass to CMake") @click.option('--shared', is_flag=True, help="Set toolchain to build shared libraries by default") @click.option('--static', is_flag=True, help="Set toolchain to build static libraries by default") -def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, define, shared, static): +@click.option('--no-global-include', is_flag=True, help="Don't use global include dir (required for -X header)") +def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, define, shared, static, no_global_include): """ Initialize install directory """ if shared and static: click.echo("ERROR: shared and static are not supported together") @@ -73,7 +74,9 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def cxxflags=cxxflags, ldflags=ldflags, std=std, - defines=defines) + defines=defines, + no_global_include=no_global_include + ) @cli.command(name='install') @use_prefix diff --git a/cget/prefix.py b/cget/prefix.py index ad8e658..ef53142 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -117,20 +117,32 @@ def write_cmake(self, always_write=False, **kwargs): @returns(inspect.isgenerator) @util.yield_from - def generate_cmake_toolchain(self, toolchain=None, cc=None, cxx=None, cflags=None, cxxflags=None, ldflags=None, std=None, defines=None): + def generate_cmake_toolchain( + self, + toolchain=None, + cc=None, + cxx=None, + cflags=None, + cxxflags=None, + ldflags=None, + std=None, + defines=None, + no_global_include=False + ): set_ = cmake_set if_ = cmake_if else_ = cmake_else append_ = cmake_append yield set_('CGET_PREFIX', self.prefix) yield set_('CMAKE_PREFIX_PATH', self.prefix) - yield if_('${CMAKE_VERSION} VERSION_LESS "3.6.0"', - ['include_directories(SYSTEM ${CGET_PREFIX}/include)'], - else_( - set_('CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include'), - set_('CMAKE_C_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include') + if not no_global_include: + yield if_('${CMAKE_VERSION} VERSION_LESS "3.6.0"', + ['include_directories(SYSTEM ${CGET_PREFIX}/include)'], + else_( + set_('CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include'), + set_('CMAKE_C_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include') + ) ) - ) if toolchain: yield ['include({})'.format(util.quote(os.path.abspath(toolchain)))] yield if_('CMAKE_CROSSCOMPILING', append_('CMAKE_FIND_ROOT_PATH', self.prefix) From dcb990f771c4b5428be1c7ca1db8e162d1c0a109 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 10:07:20 +0100 Subject: [PATCH 14/48] add dirhash dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 1df73e3..0d55ef5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ click>=6.6 # PyYAML six>=1.10 +dirhash>=0.2.1 From 4366aff60a52bf748576688075b65c172625b7da Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 10:07:20 +0100 Subject: [PATCH 15/48] add dirhash dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2e2a871..0e29bfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ click>=6.6 # PyYAML six>=1.10 requests +dirhash>=0.2.1 From 0ddf42618cf57befad97fea316760f13599b2ef1 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 10:58:51 +0100 Subject: [PATCH 16/48] get builds from cache if possible --- cget/cli.py | 2 -- cget/prefix.py | 35 +++++++++++++++++++++-------------- cget/util.py | 19 +++++++++++++++---- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index f321492..886dcb1 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -100,8 +100,6 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato else: file = 'requirements.txt' pbs = [PackageBuild(pkg, cmake=cmake, variant=variant) for pkg in pkgs] for pbu in util.flat([prefix.from_file(file), pbs]): - hash = prefix.hash_pkg(pbu) - print("package '%s' hash %s" % (prefix.parse_pkg_src(pbu).name, hash)) pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) diff --git a/cget/prefix.py b/cget/prefix.py index 8d50e86..0d7ff9f 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -320,25 +320,32 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, self.write_parent(pb, track=track) if update: self.remove(pb) else: return "Package {} already installed".format(pb.to_name()) + package_hash = self.hash_pkg(pb) + print("package %s hash %s" % (pb.to_name(), package_hash)) + #return "dry run" + build_cache_prefix = "builds/%s" % pb.to_name() with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: # Fetch package src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) # Install any dependencies first self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) - util.zip_dir_to_cache("builds/%s" % pb.to_name(), self.hash_pkg(pb), install_dir) + if util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) + util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/cget/util.py b/cget/util.py index 6ef75aa..3ea77b8 100644 --- a/cget/util.py +++ b/cget/util.py @@ -96,15 +96,26 @@ def zipdir(src_dir, tgt_file): os.path.join(root, file), os.path.relpath( os.path.join(root, file), - os.path.join(src_dir, '..') + os.path.join(src_dir) ) ) zipf.close() def zip_dir_to_cache(prefix, key, src_dir): - out_dir = get_cache_path(prefix) - mkdir(out_dir) - zipdir(src_dir, os.path.join(out_dir, key + ".zip")) + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + mkdir(cache_dir) + zipdir(src_dir, zipfile_path) + +def unzip_dir_from_cache(prefix, key, tgt_dir): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + if os.path.exists(zipfile_path): + f = zipfile.ZipFile(zipfile_path, "r") + f.extractall(tgt_dir) + return True + else: + return False def ls(p, predicate=lambda x:True): if os.path.exists(p): From 90015b27964e2ab266fd28f4dab4c6aa2bd68681 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 11:10:29 +0100 Subject: [PATCH 17/48] lock cache --- cget/util.py | 49 ++++++++++++++++++++++++++++-------------------- requirements.txt | 2 ++ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/cget/util.py b/cget/util.py index 3ea77b8..c7b6dea 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib, ssl, filelock if sys.version_info[0] < 3: try: @@ -87,6 +87,11 @@ def mkfile(d, file, content, always_write=True): write_to(p, content) return p +def cache_lock(): + cache_base_dir = get_cache_path() + mkdir(cache_base_dir) + return filelock.FileLock(os.path.join(cache_base_dir, "lock")) + def zipdir(src_dir, tgt_file): print("zipping '%s' to '%s" % (src_dir, tgt_file)) zipf = zipfile.ZipFile(tgt_file, 'w', zipfile.ZIP_DEFLATED) @@ -102,20 +107,22 @@ def zipdir(src_dir, tgt_file): zipf.close() def zip_dir_to_cache(prefix, key, src_dir): - cache_dir = get_cache_path(prefix) - zipfile_path = os.path.join(cache_dir, key + ".zip") - mkdir(cache_dir) - zipdir(src_dir, zipfile_path) + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + mkdir(cache_dir) + zipdir(src_dir, zipfile_path) def unzip_dir_from_cache(prefix, key, tgt_dir): - cache_dir = get_cache_path(prefix) - zipfile_path = os.path.join(cache_dir, key + ".zip") - if os.path.exists(zipfile_path): - f = zipfile.ZipFile(zipfile_path, "r") - f.extractall(tgt_dir) - return True - else: - return False + with cache_lock(): + cache_dir = get_cache_path(prefix) + zipfile_path = os.path.join(cache_dir, key + ".zip") + if os.path.exists(zipfile_path): + f = zipfile.ZipFile(zipfile_path, "r") + f.extractall(tgt_dir) + return True + else: + return False def ls(p, predicate=lambda x:True): if os.path.exists(p): @@ -136,15 +143,17 @@ def adjust_path(p): return p def add_cache_file(key, f): - mkdir(get_cache_path(key)) - shutil.copy2(f, get_cache_path(key, os.path.basename(f))) + with cache_lock(): + mkdir(get_cache_path(key)) + shutil.copy2(f, get_cache_path(key, os.path.basename(f))) def get_cache_file(key): - p = get_cache_path(key) - if os.path.exists(p): - return os.path.join(p, next(ls(p))) - else: - return None + with cache_lock(): + p = get_cache_path(key) + if os.path.exists(p): + return os.path.join(p, next(ls(p))) + else: + return None def delete_dir(path): if path is not None and os.path.exists(path): shutil.rmtree(adjust_path(path)) diff --git a/requirements.txt b/requirements.txt index 0d55ef5..2f15aa5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ click>=6.6 # PyYAML six>=1.10 dirhash>=0.2.1 +filelock>=2.0.13 + From 41bd679d736c257f0508908dc89b78541ef08fef Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 11:26:28 +0100 Subject: [PATCH 18/48] make build cache optional --- cget/cli.py | 13 +++++++++++-- cget/prefix.py | 7 ++++--- requirements.txt | 1 - 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index 886dcb1..f499dd4 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -87,8 +87,9 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def @click.option('--debug', is_flag=True, help="Install debug version") @click.option('--release', is_flag=True, help="Install release version") @click.option('--insecure', is_flag=True, help="Don't use https urls") +@click.option('--use-build-cache', is_flag=True, help="Cache builds") @click.argument('pkgs', nargs=-1, type=click.STRING) -def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure): +def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache): """ Install packages """ if debug and release: click.echo("ERROR: debug and release are not supported together") @@ -102,7 +103,15 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato for pbu in util.flat([prefix.from_file(file), pbs]): pb = pbu.merge_defines(define) with prefix.try_("Failed to build package {}".format(pb.to_name()), on_fail=lambda: prefix.remove(pb)): - click.echo(prefix.install(pb, test=test, test_all=test_all, update=update, generator=generator, insecure=insecure)) + click.echo(prefix.install( + pb, + test=test, + test_all=test_all, + update=update, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache + )) @cli.command(name='ignore') @use_prefix diff --git a/cget/prefix.py b/cget/prefix.py index 0d7ff9f..fe2dd92 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -304,7 +304,7 @@ def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecu @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) - def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False): + def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False, use_build_cache=False): pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) @@ -329,7 +329,7 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) # Install any dependencies first self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) - if util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): print("retreived Package {} from cache".format(pb.to_name())) else: # Setup cmake file @@ -345,7 +345,8 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, if test or test_all: builder.test(variant=pb.variant) # Install builder.build(target='install', variant=pb.variant) - util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) + if use_build_cache: + util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/requirements.txt b/requirements.txt index 2f15aa5..4948a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ click>=6.6 six>=1.10 dirhash>=0.2.1 filelock>=2.0.13 - From 4478066efbb836c7818fb8f24e251d367593bb2b Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 14:31:24 +0100 Subject: [PATCH 19/48] propagate build cache setting to dependencies --- cget/prefix.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index fe2dd92..3ff72c0 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -294,13 +294,20 @@ def from_file(self, file, url=None, no_recipe=False): def write_parent(self, pb, track=True): if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent) - def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False): + def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False, use_build_cache=False): for dependent in self.from_file(pb.requirements or os.path.join(d, 'requirements.txt'), pb.pkg_src.url): transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing if installable: - self.install(dependent.of(pb), test_all=test_all, generator=generator, track=not transient, insecure=insecure) + self.install( + dependent.of(pb), + test_all=test_all, + generator=generator, + track=not transient, + insecure=insecure, + use_build_cache=use_build_cache + ) @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) @@ -328,7 +335,15 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, # Fetch package src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) # Install any dependencies first - self.install_deps(pb, src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure) + self.install_deps( + pb, + src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache + ) if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): print("retreived Package {} from cache".format(pb.to_name())) else: From e5021aa325ced6a6bd18c12f5a71a58f5dfb253b Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 15:03:54 +0100 Subject: [PATCH 20/48] allow fast path cached builds by ignoring in source deps --- cget/cli.py | 6 ++- cget/prefix.py | 102 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/cget/cli.py b/cget/cli.py index f499dd4..0bf8363 100644 --- a/cget/cli.py +++ b/cget/cli.py @@ -88,8 +88,9 @@ def init_command(prefix, toolchain, cc, cxx, cflags, cxxflags, ldflags, std, def @click.option('--release', is_flag=True, help="Install release version") @click.option('--insecure', is_flag=True, help="Don't use https urls") @click.option('--use-build-cache', is_flag=True, help="Cache builds") +@click.option('--recipe-deps-only', is_flag=True, help="only use dependencies from recipes (speeds up cached builds a lot)") @click.argument('pkgs', nargs=-1, type=click.STRING) -def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache): +def install_command(prefix, pkgs, define, file, test, test_all, update, generator, cmake, debug, release, insecure, use_build_cache, recipe_deps_only): """ Install packages """ if debug and release: click.echo("ERROR: debug and release are not supported together") @@ -110,7 +111,8 @@ def install_command(prefix, pkgs, define, file, test, test_all, update, generato update=update, generator=generator, insecure=insecure, - use_build_cache=use_build_cache + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only )) @cli.command(name='ignore') diff --git a/cget/prefix.py b/cget/prefix.py index 3ff72c0..bc0bfc5 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -294,8 +294,24 @@ def from_file(self, file, url=None, no_recipe=False): def write_parent(self, pb, track=True): if track and pb.parent is not None: util.mkfile(self.get_deps_directory(pb.to_fname()), pb.parent, pb.parent) - def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecure=False, use_build_cache=False): - for dependent in self.from_file(pb.requirements or os.path.join(d, 'requirements.txt'), pb.pkg_src.url): + def install_deps( + self, + pb, + src_dir=None, + test=False, + test_all=False, + generator=None, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): + if pb.requirements: + dependents = self.from_file(pb.requirements, pb.pkg_src.url) + elif src_dir: + dependents = self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) + else: + return + for dependent in dependents: transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing @@ -306,12 +322,24 @@ def install_deps(self, pb, d, test=False, test_all=False, generator=None, insecu generator=generator, track=not transient, insecure=insecure, - use_build_cache=use_build_cache + use_build_cache=use_build_cache, + recipe_deps_only=recipe_deps_only ) @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) - def install(self, pb, test=False, test_all=False, generator=None, update=False, track=True, insecure=False, use_build_cache=False): + def install( + self, + pb, + test=False, + test_all=False, + generator=None, + update=False, + track=True, + insecure=False, + use_build_cache=False, + recipe_deps_only=False + ): pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) @@ -329,41 +357,57 @@ def install(self, pb, test=False, test_all=False, generator=None, update=False, else: return "Package {} already installed".format(pb.to_name()) package_hash = self.hash_pkg(pb) print("package %s hash %s" % (pb.to_name(), package_hash)) - #return "dry run" build_cache_prefix = "builds/%s" % pb.to_name() - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first + need_build = True + if recipe_deps_only: self.install_deps( pb, - src_dir, test=test, test_all=test_all, generator=generator, insecure=insecure, - use_build_cache=use_build_cache + use_build_cache=use_build_cache, + recipe_deps_only=True ) if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): print("retreived Package {} from cache".format(pb.to_name())) - else: - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) - if use_build_cache: - util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) - if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) - else: util.copy_dir(install_dir, self.prefix) + need_build = False + if need_build: + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False + ) + if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) + if use_build_cache: + util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) + if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) + else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) return "Successfully installed {}".format(pb.to_name()) From 7e17de7bb097846cb6b1300076e6def4e43d3f38 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 12 Jan 2021 12:10:20 +0100 Subject: [PATCH 21/48] preserve recipe in package source for hashing --- cget/package.py | 2 ++ cget/prefix.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cget/package.py b/cget/package.py index 649829e..99737b9 100644 --- a/cget/package.py +++ b/cget/package.py @@ -33,8 +33,10 @@ def get_src_dir(self): def calc_hash(self): if self.recipe: + print("calculating dirshash of recipe '%s' package '%s'" % (self.recipe, self.to_name())) return dirhash.dirhash(self.recipe, "sha1") elif self.url: + print("calculating hash of url '%s' package '%s'" % (self.url, self.to_name())) return hashlib.sha1(self.url.encode("utf-8")).hexdigest() raise Exception("no url or recipe: %s" % self.__dict__) diff --git a/cget/prefix.py b/cget/prefix.py index 6b98370..af15197 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -217,7 +217,7 @@ def get_unlink_deps_directory(self, name, *dirs): def parse_src_file(self, name, url, start=None): f = util.actual_path(url, start) self.log('parse_src_file actual_path:', start, f) - if os.path.exists(f): return PackageSource(name=name, url='file://' + f) + if os.path.isfile(f): return PackageSource(name=name, url='file://' + f) return None def parse_src_recipe(self, name, url): @@ -260,14 +260,14 @@ def parse_pkg_src(self, pkg, start=None, no_recipe=False): @returns(PackageBuild) @params(pkg=PACKAGE_SOURCE_TYPES) def parse_pkg_build(self, pkg, start=None, no_recipe=False): - if isinstance(pkg, PackageBuild): + if isinstance(pkg, PackageBuild): pkg.pkg_src = self.parse_pkg_src(pkg.pkg_src, start, no_recipe) if pkg.pkg_src.recipe: pkg = self.from_recipe(pkg.pkg_src.recipe, pkg) if pkg.cmake: pkg.cmake = find_cmake(pkg.cmake, start) return pkg else: pkg_src = self.parse_pkg_src(pkg, start, no_recipe) - if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, pkg_src.name) + if pkg_src.recipe: return self.from_recipe(pkg_src.recipe, name=pkg_src.name) else: return PackageBuild(pkg_src) def from_recipe(self, recipe, pkg=None, name=None): @@ -277,7 +277,7 @@ def from_recipe(self, recipe, pkg=None, name=None): self.check(lambda:p.pkg_src is not None) requirements = os.path.join(recipe, "requirements.txt") if os.path.exists(requirements): p.requirements = requirements - p.pkg_src.recipe = None + p.pkg_src.recipe = recipe # Use original name if pkg: p.pkg_src.name = pkg.pkg_src.name elif name: p.pkg_src.name = name From 72c0eafdb886b0bd699c647e63eb47710c6f562e Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 12 Jan 2021 16:33:04 +0100 Subject: [PATCH 22/48] prevent cache access from crashing --- cget/util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cget/util.py b/cget/util.py index 9ad121f..f8dedf1 100644 --- a/cget/util.py +++ b/cget/util.py @@ -151,9 +151,10 @@ def get_cache_file(key): with cache_lock(): p = get_cache_path(key) if os.path.exists(p): - return os.path.join(p, next(ls(p))) - else: - return None + content = list(ls(p)) + if content: + return os.path.join(p, content[0]) + return None def delete_dir(path): if path is not None and os.path.exists(path): shutil.rmtree(adjust_path(path)) From 4247383138b9e6bc529ad5a42e7f267dffc0f74b Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 12 Jan 2021 16:39:33 +0100 Subject: [PATCH 23/48] show non matching hash value --- cget/util.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cget/util.py b/cget/util.py index f8dedf1..7d8f6cf 100644 --- a/cget/util.py +++ b/cget/util.py @@ -280,10 +280,12 @@ def retrieve_url(url, dst, copy=False, insecure=False, hash=None): f = download_to(url, dst, insecure=insecure) if remote else transfer_to(url[7:], dst, copy=copy) if os.path.isfile(f) and hash: click.echo("Computing hash: {}".format(hash)) - if check_hash(f, hash): + hash_type, hash_value = hash.lower().split(':') + computed = hash_file(f, hash_type) + if computed == hash_value: if remote: add_cache_file(hash.replace(':', '-'), f) else: - raise BuildError("Hash doesn't match for {0}: {1}".format(url, hash)) + raise BuildError("Hash doesn't match for {0}: {1} != {2}".format(url, hash_type + ":" + computed, hash)) return f def extract_ar(archive, dst, *kwargs): @@ -310,10 +312,6 @@ def hash_file(f, t): h.update(open(f, 'rb').read()) return h.hexdigest() -def check_hash(f, hash): - t, h = hash.lower().split(':') - return hash_file(f, t) == h - def which(p, paths=None, throws=True): exes = [p+x for x in ['', '.exe', '.bat']] for dirname in list(paths or [])+os.environ['PATH'].split(os.pathsep): From dfc28db682afc723be2cfbff1715ff70cca81058 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 12 Jan 2021 17:26:30 +0100 Subject: [PATCH 24/48] add option to autotools.cmake to run autogen.sh --- cget/cmake/autotools.cmake | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index 3701785..aeccc00 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -181,7 +181,18 @@ list(APPEND CONFIGURE_OPTIONS ) endif() -execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS) +set(AUTOTOOLS_RUN_AUTOGEN_SH Off CACHE BOOL "") +if (AUTOTOOLS_RUN_AUTOGEN_SH) + execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/autogen.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endif (AUTOTOOLS_RUN_AUTOGEN_SH) + +execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help + OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS +) if(AUTOTOOLS_AVAILABLE_OPTIONS MATCHES "--disable-option-checking") set(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS On CACHE BOOL "") From 12becfb5a7a2ee76a69f2b2abe349163d80b115a Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Wed, 13 Jan 2021 16:58:17 +0100 Subject: [PATCH 25/48] add optional boost bootstrap args --- cget/cmake/boost.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 87127ec..a4a79e3 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -242,13 +242,13 @@ find_program(B2_EXE b2) if(NOT ${B2_EXE}) if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") From 887b08d51af5c3b18beeb69074eb99d47b1d093c Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 11:57:28 +0100 Subject: [PATCH 26/48] stable setting of python exe for boost --- cget/cmake/autotools.cmake | 6 ++++-- cget/cmake/boost.cmake | 6 ++++++ tools/cmake/autotools.cmake | 15 ++++++++++++++- tools/cmake/boost.cmake | 10 ++++++++-- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index aeccc00..2dd1b27 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -181,6 +181,7 @@ list(APPEND CONFIGURE_OPTIONS ) endif() + set(AUTOTOOLS_RUN_AUTOGEN_SH Off CACHE BOOL "") if (AUTOTOOLS_RUN_AUTOGEN_SH) execute_process( @@ -190,8 +191,8 @@ if (AUTOTOOLS_RUN_AUTOGEN_SH) endif (AUTOTOOLS_RUN_AUTOGEN_SH) execute_process( - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help - OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help + OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS ) if(AUTOTOOLS_AVAILABLE_OPTIONS MATCHES "--disable-option-checking") @@ -211,6 +212,7 @@ if(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS) endif() endif() + message(STATUS "Configure options: ${CONFIGURE_OPTIONS}") # TODO: Check flags of configure script diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index a4a79e3..5113ab8 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -240,6 +240,12 @@ file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") find_program(B2_EXE b2) if(NOT ${B2_EXE}) + set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") + set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") + if (BOOST_PYTHON) + file(REAL_PATH ${BOOST_PYTHON} + set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") + endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} diff --git a/tools/cmake/autotools.cmake b/tools/cmake/autotools.cmake index 2629822..45c7ebd 100644 --- a/tools/cmake/autotools.cmake +++ b/tools/cmake/autotools.cmake @@ -34,7 +34,19 @@ list(APPEND CONFIGURE_OPTIONS ) endif() -execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS) + +set(AUTOTOOLS_RUN_AUTOGEN_SH Off CACHE BOOL "") +if (AUTOTOOLS_RUN_AUTOGEN_SH) + execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/autogen.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endif (AUTOTOOLS_RUN_AUTOGEN_SH) + +execute_process( + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure --help + OUTPUT_VARIABLE AUTOTOOLS_AVAILABLE_OPTIONS +) if(AUTOTOOLS_AVAILABLE_OPTIONS MATCHES "--disable-option-checking") set(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS On CACHE BOOL "") @@ -53,6 +65,7 @@ if(AUTOTOOL_IMPLICIT_CONFIGURE_OPTIONS) endif() endif() + message(STATUS "Configure options: ${CONFIGURE_OPTIONS}") # TODO: Check flags of configure script diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index 3337730..7d89bef 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -93,15 +93,21 @@ file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") find_program(B2_EXE b2) if(NOT ${B2_EXE}) + set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") + set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") + if (BOOST_PYTHON) + file(REAL_PATH ${BOOST_PYTHON} + set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") + endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") From b03e046a919350a6b3a5877412035964a3d2de8c Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 12:20:11 +0100 Subject: [PATCH 27/48] preserve file permissions in unzip --- cget/util.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cget/util.py b/cget/util.py index 7d8f6cf..a464f2e 100644 --- a/cget/util.py +++ b/cget/util.py @@ -106,6 +106,16 @@ def zipdir(src_dir, tgt_file): ) zipf.close() +def unzip(zipname, extract_dir): + def extract_file(zf, info, extract_dir): + zf.extract( info.filename, path=extract_dir ) + out_path = os.path.join( extract_dir, info.filename ) + perm = info.external_attr >> 16L + os.chmod( out_path, perm ) + with zipfile.ZipFile(zipname, 'r') as zf: + for info in zf.infolist(): + extract_file(zf, info, extract_dir) + def zip_dir_to_cache(prefix, key, src_dir): with cache_lock(): cache_dir = get_cache_path(prefix) @@ -118,8 +128,7 @@ def unzip_dir_from_cache(prefix, key, tgt_dir): cache_dir = get_cache_path(prefix) zipfile_path = os.path.join(cache_dir, key + ".zip") if os.path.exists(zipfile_path): - f = zipfile.ZipFile(zipfile_path, "r") - f.extractall(tgt_dir) + unzip(zipfile_path, tgt_dir) return True else: return False From c60ca421981fa3df4a6f4a2885aa9dfc366484f4 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 12:31:35 +0100 Subject: [PATCH 28/48] fix python syntax --- cget/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cget/util.py b/cget/util.py index a464f2e..1d8b2fa 100644 --- a/cget/util.py +++ b/cget/util.py @@ -110,7 +110,7 @@ def unzip(zipname, extract_dir): def extract_file(zf, info, extract_dir): zf.extract( info.filename, path=extract_dir ) out_path = os.path.join( extract_dir, info.filename ) - perm = info.external_attr >> 16L + perm = info.external_attr >> 16 os.chmod( out_path, perm ) with zipfile.ZipFile(zipname, 'r') as zf: for info in zf.infolist(): From 7011815ef6375f0d510bd0cb5be55ccd963f46cb Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 12:35:20 +0100 Subject: [PATCH 29/48] fix boost.cmake --- cget/cmake/boost.cmake | 2 +- tools/cmake/boost.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 5113ab8..a1e2d47 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -243,7 +243,7 @@ if(NOT ${B2_EXE}) set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) - file(REAL_PATH ${BOOST_PYTHON} + file(REAL_PATH ${BOOST_PYTHON}) set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") endif() if(CMAKE_HOST_WIN32) diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index 7d89bef..7859868 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -96,7 +96,7 @@ if(NOT ${B2_EXE}) set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) - file(REAL_PATH ${BOOST_PYTHON} + file(REAL_PATH ${BOOST_PYTHON}) set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") endif() if(CMAKE_HOST_WIN32) From 8ee6cf4252dfe4975f8a8270f6c413a017faf69c Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 12:43:36 +0100 Subject: [PATCH 30/48] support cmake < 3.19 --- cget/cmake/boost.cmake | 2 +- tools/cmake/boost.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index a1e2d47..0eeb033 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -243,7 +243,7 @@ if(NOT ${B2_EXE}) set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) - file(REAL_PATH ${BOOST_PYTHON}) + get_filename_component(BOOST_PYTHON ${BOOST_PYTHON} REALPATH) set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") endif() if(CMAKE_HOST_WIN32) diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index 7859868..225ce15 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -96,7 +96,7 @@ if(NOT ${B2_EXE}) set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) - file(REAL_PATH ${BOOST_PYTHON}) + get_filename_component(BOOST_PYTHON ${BOOST_PYTHON} REALPATH) set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") endif() if(CMAKE_HOST_WIN32) From e723cf022be9f0c306bc530459559b6613f6b1b1 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 13:33:18 +0100 Subject: [PATCH 31/48] fix boost python discovery --- cget/cmake/boost.cmake | 14 ++++++++------ tools/cmake/boost.cmake | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 0eeb033..db1c5ce 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -238,23 +238,25 @@ message("${B2_CONFIG_CONTENT}") file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") +set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") +set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") + find_program(B2_EXE b2) if(NOT ${B2_EXE}) - set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") - set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) - get_filename_component(BOOST_PYTHON ${BOOST_PYTHON} REALPATH) - set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") + find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}'") + set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND}") endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index 225ce15..3232e62 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -91,23 +91,25 @@ message("${B2_CONFIG_CONTENT}") file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") +set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") +set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") + find_program(B2_EXE b2) if(NOT ${B2_EXE}) - set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") - set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) - get_filename_component(BOOST_PYTHON ${BOOST_PYTHON} REALPATH) - set(BOOST_BOOTSTRAP_ARGS "${BOOST_BOOTSTRAP_ARGS} --with-python=${BOOST_PYTHON}") + find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}'") + set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND}") endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") From a41cd2c22ff451b8a538b451761edbfd9488b204 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 14:17:08 +0100 Subject: [PATCH 32/48] find python by absolute path for boost --- cget/cmake/boost.cmake | 5 +++-- tools/cmake/boost.cmake | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index db1c5ce..b099074 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -245,8 +245,9 @@ find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) - message(STATUS "found python: '${BOOST_PYTHON_FOUND}'") - set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND}") + get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") + set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND_REAL}") endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index 3232e62..a5a7288 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -98,8 +98,9 @@ find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) - message(STATUS "found python: '${BOOST_PYTHON_FOUND}'") - set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND}") + get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") + set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND_REAL}") endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap From 1679edce9b854903ff081b156b1f1e41d9726cb8 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 14 Jan 2021 14:57:29 +0100 Subject: [PATCH 33/48] use top level bootstrap in boost.cmake --- cget/cmake/boost.cmake | 10 +++++----- tools/cmake/boost.cmake | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index b099074..69b07f1 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -251,16 +251,16 @@ if(NOT ${B2_EXE}) endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) - set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") + set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/b2") endif() install(PROGRAMS ${B2_EXE} DESTINATION bin) endif() diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index a5a7288..f0886a7 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -104,16 +104,16 @@ if(NOT ${B2_EXE}) endif() if(CMAKE_HOST_WIN32) add_custom_target(bootstrap - COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2.exe") else() add_custom_target(bootstrap - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/build/ + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.sh ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) - set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/tools/build/b2") + set(B2_EXE "${CMAKE_CURRENT_SOURCE_DIR}/b2") endif() install(PROGRAMS ${B2_EXE} DESTINATION bin) endif() From 750aab19e3991a5db041c88c8b72b856f09af76f Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 15 Jan 2021 13:23:20 +0100 Subject: [PATCH 34/48] dont overwrite linker flags repeatedly --- cget/cmake/autotools.cmake | 3 +-- cget/cmake/boost.cmake | 3 +-- cget/cmake/make.cmake | 3 +-- cget/cmake/meson.cmake | 3 +-- tools/preamble.cmake | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index 2dd1b27..6c94dce 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -86,6 +86,7 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") @@ -112,13 +113,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 69b07f1..63dc430 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -79,6 +79,7 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") @@ -105,13 +106,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) diff --git a/cget/cmake/make.cmake b/cget/cmake/make.cmake index 7d5f495..56c339f 100644 --- a/cget/cmake/make.cmake +++ b/cget/cmake/make.cmake @@ -84,6 +84,7 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") @@ -110,13 +111,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) diff --git a/cget/cmake/meson.cmake b/cget/cmake/meson.cmake index 73e4601..cc2fe32 100644 --- a/cget/cmake/meson.cmake +++ b/cget/cmake/meson.cmake @@ -113,6 +113,7 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") @@ -139,13 +140,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) diff --git a/tools/preamble.cmake b/tools/preamble.cmake index ae1fed6..16c08fe 100644 --- a/tools/preamble.cmake +++ b/tools/preamble.cmake @@ -69,6 +69,7 @@ macro(preamble PREFIX) endif() endforeach() get_directory_property(${PREFIX}_LINK_DIRECTORIES LINK_DIRECTORIES) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LIB_DIR ${${PREFIX}_LINK_DIRECTORIES}) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") @@ -95,13 +96,11 @@ macro(preamble PREFIX) set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") endif (APPLE) - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") else() string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_STATIC_LINKER_FLAGS}") endif() - get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) From de35535b303b3568434d60277b4059a0841b831b Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 18 Jan 2021 09:58:26 +0100 Subject: [PATCH 35/48] remove debug log --- cget/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cget/package.py b/cget/package.py index 99737b9..649829e 100644 --- a/cget/package.py +++ b/cget/package.py @@ -33,10 +33,8 @@ def get_src_dir(self): def calc_hash(self): if self.recipe: - print("calculating dirshash of recipe '%s' package '%s'" % (self.recipe, self.to_name())) return dirhash.dirhash(self.recipe, "sha1") elif self.url: - print("calculating hash of url '%s' package '%s'" % (self.url, self.to_name())) return hashlib.sha1(self.url.encode("utf-8")).hexdigest() raise Exception("no url or recipe: %s" % self.__dict__) From 3bd0bed782d8c4f559bd8360c8b1726a71922daa Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 18 Jan 2021 09:59:43 +0100 Subject: [PATCH 36/48] move debug out to log --- cget/prefix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cget/prefix.py b/cget/prefix.py index af15197..9a0ebd4 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -368,7 +368,7 @@ def install( if update: self.remove(pb) else: return "Package {} already installed".format(pb.to_name()) package_hash = self.hash_pkg(pb) - print("package %s hash %s" % (pb.to_name(), package_hash)) + self.log("package %s hash %s" % (pb.to_name(), package_hash)) build_cache_prefix = "builds/%s" % pb.to_name() need_build = True if recipe_deps_only: From 5e754d2ad0f8c8a0d5bc668f1b4640532f42adee Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 18 Jan 2021 13:42:11 +0100 Subject: [PATCH 37/48] double python hint for boost on osx --- cget/cmake/boost.cmake | 6 ++++++ tools/cmake/boost.cmake | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 63dc430..7c5a7d4 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -240,6 +240,12 @@ file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") +if (BOOST_PYTHON) + set(B2_CONFIG_CONTENT "${B2_CONFIG_CONTENT} + using python : : ${BOOST_PYTHON} ; + ") +endif (BOOST_PYTHON) + find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index f0886a7..a2e8582 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -94,6 +94,12 @@ file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") +if (BOOST_PYTHON) + set(B2_CONFIG_CONTENT "${B2_CONFIG_CONTENT} + using python : : ${BOOST_PYTHON} ; + ") +endif (BOOST_PYTHON) + find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) From 4929807f222d7970ce8fa14d2b95ea2f97ecfcc0 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 18 Jan 2021 13:45:17 +0100 Subject: [PATCH 38/48] emit final boost user-config --- cget/cmake/boost.cmake | 7 ++++--- tools/cmake/boost.cmake | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 7c5a7d4..f8c8bb2 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -233,9 +233,6 @@ using ${BOOST_TOOLCHAIN} : ${B2_TOOLCHAIN_VERSION} : \"${B2_COMPILER}\" : ${SEARCH_PATHS} ; ") -message("${B2_CONFIG_CONTENT}") - -file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") @@ -246,6 +243,10 @@ if (BOOST_PYTHON) ") endif (BOOST_PYTHON) +message("${B2_CONFIG_CONTENT}") + +file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") + find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index a2e8582..2433f3d 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -87,9 +87,6 @@ using ${BOOST_TOOLCHAIN} : ${B2_TOOLCHAIN_VERSION} : \"${B2_COMPILER}\" : ${SEARCH_PATHS} ; ") -message("${B2_CONFIG_CONTENT}") - -file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") @@ -100,6 +97,10 @@ if (BOOST_PYTHON) ") endif (BOOST_PYTHON) +message("${B2_CONFIG_CONTENT}") + +file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") + find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) From acf16d5c124bf0a5d967244afa3512de7a7a9467 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 18 Jan 2021 14:12:57 +0100 Subject: [PATCH 39/48] dont put space after -L --- cget/cmake/autotools.cmake | 2 +- cget/cmake/boost.cmake | 2 +- cget/cmake/make.cmake | 2 +- cget/cmake/meson.cmake | 2 +- tools/preamble.cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index 6c94dce..143a841 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -91,7 +91,7 @@ macro(preamble PREFIX) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index f8c8bb2..f958f4f 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -84,7 +84,7 @@ macro(preamble PREFIX) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() diff --git a/cget/cmake/make.cmake b/cget/cmake/make.cmake index 56c339f..f4fdd5a 100644 --- a/cget/cmake/make.cmake +++ b/cget/cmake/make.cmake @@ -89,7 +89,7 @@ macro(preamble PREFIX) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() diff --git a/cget/cmake/meson.cmake b/cget/cmake/meson.cmake index cc2fe32..8a82a6e 100644 --- a/cget/cmake/meson.cmake +++ b/cget/cmake/meson.cmake @@ -118,7 +118,7 @@ macro(preamble PREFIX) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() diff --git a/tools/preamble.cmake b/tools/preamble.cmake index 16c08fe..b95f76f 100644 --- a/tools/preamble.cmake +++ b/tools/preamble.cmake @@ -74,7 +74,7 @@ macro(preamble PREFIX) if(MSVC) string(APPEND ${PREFIX}_LINK_FLAGS " /LIBPATH:${LIB_DIR}") else() - string(APPEND ${PREFIX}_LINK_FLAGS " -L ${LIB_DIR}") + string(APPEND ${PREFIX}_LINK_FLAGS " -L${LIB_DIR}") endif() endforeach() From 8fc1c2955263ef779dcee56e1a21ce58353433f3 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 13:43:14 +0100 Subject: [PATCH 40/48] fix build cache --- cget/prefix.py | 60 ++++++++++++++++++++++++++++---------------------- cget/util.py | 2 +- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index 9a0ebd4..79f1eba 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -355,7 +355,6 @@ def install( pb = self.parse_pkg_build(pb) pkg_dir = self.get_package_directory(pb.to_fname()) unlink_dir = self.get_unlink_directory(pb.to_fname()) - install_dir = self.get_package_directory(pb.to_fname(), 'install') # If its been unlinked, then link it in if os.path.exists(unlink_dir): if update: shutil.rmtree(unlink_dir) @@ -369,8 +368,15 @@ def install( else: return "Package {} already installed".format(pb.to_name()) package_hash = self.hash_pkg(pb) self.log("package %s hash %s" % (pb.to_name(), package_hash)) - build_cache_prefix = "builds/%s" % pb.to_name() - need_build = True + pkg_install_dir = self.get_package_directory(pb.to_fname(), 'install') + if use_build_cache: + install_dir = util.get_cache_path("builds", pb.to_name(), package_hash) + util.mkdir(pkg_dir) + os.symlink(install_dir, pkg_install_dir) + self.log("using cached install dir '%s'" % install_dir) + else: + install_dir = pkg_install_dir + self.log("using local install dir '%s'" % install_dir) if recipe_deps_only: self.install_deps( pb, @@ -381,26 +387,23 @@ def install( use_build_cache=use_build_cache, recipe_deps_only=True ) - if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): - print("retreived Package {} from cache".format(pb.to_name())) - need_build = False - if need_build: - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first - if not recipe_deps_only: - self.install_deps( - pb, - src_dir=src_dir, - test=test, - test_all=test_all, - generator=generator, - insecure=insecure, - use_build_cache=use_build_cache, - recipe_deps_only=False - ) - if not update and use_build_cache and util.unzip_dir_from_cache(build_cache_prefix, package_hash, install_dir): + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, + test=test, + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False + ) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): print("retreived Package {} from cache".format(pb.to_name())) else: # Setup cmake file @@ -410,14 +413,19 @@ def install( os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) shutil.copyfile(pb.cmake, target) # Configure and build - builder.configure(src_dir, defines=pb.define, generator=generator, install_prefix=install_dir, test=test, variant=pb.variant) + builder.configure( + src_dir, + defines=pb.define, + generator=generator, + install_prefix=install_dir, + test=test, + variant=pb.variant + ) builder.build(variant=pb.variant) # Run tests if enabled if test or test_all: builder.test(variant=pb.variant) # Install builder.build(target='install', variant=pb.variant) - if use_build_cache: - util.zip_dir_to_cache(build_cache_prefix, package_hash, install_dir) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) diff --git a/cget/util.py b/cget/util.py index 1d8b2fa..1eaa5c6 100644 --- a/cget/util.py +++ b/cget/util.py @@ -143,7 +143,7 @@ def get_app_dir(*args): return os.path.join(click.get_app_dir('cget'), *args) def get_cache_path(*args): - return get_app_dir('cache', *args) + return os.path.join(os.path.expanduser("~"), ".cget", "cache", *args) def adjust_path(p): # Prefixing path to avoid problems with long paths on windows From bd0d22af061964be0c83962205e7734e48549156 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 15:26:27 +0100 Subject: [PATCH 41/48] use full python path for bjam --- cget/cmake/boost.cmake | 10 +++++----- tools/cmake/boost.cmake | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index f958f4f..e6b0d52 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -238,8 +238,11 @@ set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) + find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) + get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") set(B2_CONFIG_CONTENT "${B2_CONFIG_CONTENT} - using python : : ${BOOST_PYTHON} ; + using python : : ${BOOST_PYTHON_FOUND_REAL} ; ") endif (BOOST_PYTHON) @@ -250,11 +253,8 @@ file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) - find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) - get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) - message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND_REAL}") - endif() + endif (BOOST_PYTHON) if(CMAKE_HOST_WIN32) add_custom_target(bootstrap COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} diff --git a/tools/cmake/boost.cmake b/tools/cmake/boost.cmake index 2433f3d..e5cbe1c 100644 --- a/tools/cmake/boost.cmake +++ b/tools/cmake/boost.cmake @@ -92,8 +92,11 @@ set(BOOST_PYTHON "" CACHE STRING "python executable to use for boost build") set(BOOST_BOOTSTRAP_ARGS "" CACHE STRING "additional arguments to boost bootstrap") if (BOOST_PYTHON) + find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) + get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) + message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") set(B2_CONFIG_CONTENT "${B2_CONFIG_CONTENT} - using python : : ${BOOST_PYTHON} ; + using python : : ${BOOST_PYTHON_FOUND_REAL} ; ") endif (BOOST_PYTHON) @@ -104,11 +107,8 @@ file(WRITE ${B2_CONFIG} "${B2_CONFIG_CONTENT}") find_program(B2_EXE b2) if(NOT ${B2_EXE}) if (BOOST_PYTHON) - find_program(BOOST_PYTHON_FOUND ${BOOST_PYTHON} REQUIRED) - get_filename_component(BOOST_PYTHON_FOUND_REAL "${BOOST_PYTHON_FOUND}" REALPATH) - message(STATUS "found python: '${BOOST_PYTHON_FOUND}' -> '${BOOST_PYTHON_FOUND_REAL}'") set(BOOST_BOOTSTRAP_PYTHON_ARG "--with-python=${BOOST_PYTHON_FOUND_REAL}") - endif() + endif (BOOST_PYTHON) if(CMAKE_HOST_WIN32) add_custom_target(bootstrap COMMAND cmd /c ${CMAKE_CURRENT_SOURCE_DIR}/bootstrap.bat ${BOOST_BOOTSTRAP_ARGS} ${BOOST_BOOTSTRAP_PYTHON_ARG} From bce0429654bca14139cc709ec9fcdda8387d8c04 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 16:01:11 +0100 Subject: [PATCH 42/48] avoid package fetch if possible --- cget/prefix.py | 73 +++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index 79f1eba..5433919 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -387,45 +387,50 @@ def install( use_build_cache=use_build_cache, recipe_deps_only=True ) - with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: - # Fetch package - src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) - # Install any dependencies first - if not recipe_deps_only: - self.install_deps( - pb, - src_dir=src_dir, - test=test, - test_all=test_all, - generator=generator, - insecure=insecure, - use_build_cache=use_build_cache, - recipe_deps_only=False - ) with util.cache_lock() as cache_lock: if not update and use_build_cache and os.path.exists(install_dir): print("retreived Package {} from cache".format(pb.to_name())) - else: - # Setup cmake file - if pb.cmake: - target = os.path.join(src_dir, 'CMakeLists.txt') - if os.path.exists(target): - os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) - shutil.copyfile(pb.cmake, target) - # Configure and build - builder.configure( - src_dir, - defines=pb.define, - generator=generator, - install_prefix=install_dir, + build_needed = False + if build_needed: + with self.create_builder(uuid.uuid4().hex, tmp=True) as builder: + # Fetch package + src_dir = builder.fetch(pb.pkg_src.url, pb.hash, (pb.cmake != None), insecure=insecure) + # Install any dependencies first + if not recipe_deps_only: + self.install_deps( + pb, + src_dir=src_dir, test=test, - variant=pb.variant + test_all=test_all, + generator=generator, + insecure=insecure, + use_build_cache=use_build_cache, + recipe_deps_only=False ) - builder.build(variant=pb.variant) - # Run tests if enabled - if test or test_all: builder.test(variant=pb.variant) - # Install - builder.build(target='install', variant=pb.variant) + with util.cache_lock() as cache_lock: + if not update and use_build_cache and os.path.exists(install_dir): + print("retreived Package {} from cache".format(pb.to_name())) + else: + # Setup cmake file + if pb.cmake: + target = os.path.join(src_dir, 'CMakeLists.txt') + if os.path.exists(target): + os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) + shutil.copyfile(pb.cmake, target) + # Configure and build + builder.configure( + src_dir, + defines=pb.define, + generator=generator, + install_prefix=install_dir, + test=test, + variant=pb.variant + ) + builder.build(variant=pb.variant) + # Run tests if enabled + if test or test_all: builder.test(variant=pb.variant) + # Install + builder.build(target='install', variant=pb.variant) if util.USE_SYMLINKS: util.symlink_dir(install_dir, self.prefix) else: util.copy_dir(install_dir, self.prefix) self.write_parent(pb, track=track) From a7c6fd5935497a3ee7ad04ff66ad49532ae6c61f Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 16:12:31 +0100 Subject: [PATCH 43/48] fix uninitialized variable --- cget/prefix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cget/prefix.py b/cget/prefix.py index 5433919..9180253 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -377,6 +377,7 @@ def install( else: install_dir = pkg_install_dir self.log("using local install dir '%s'" % install_dir) + build_needed = True if recipe_deps_only: self.install_deps( pb, From 8245beaf6b10ca7afda1ed41ccc32a26300468d8 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Fri, 22 Jan 2021 19:59:43 +0100 Subject: [PATCH 44/48] reduce meddling of build drivers with pkg config --- cget/cmake/autotools.cmake | 14 -------------- cget/cmake/boost.cmake | 14 -------------- cget/cmake/make.cmake | 14 -------------- cget/cmake/meson.cmake | 14 -------------- tools/preamble.cmake | 14 -------------- 5 files changed, 70 deletions(-) diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index 143a841..3db6fe0 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -60,14 +60,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -145,14 +137,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index e6b0d52..0c850f2 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -53,14 +53,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -138,14 +130,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" diff --git a/cget/cmake/make.cmake b/cget/cmake/make.cmake index f4fdd5a..28d6f22 100644 --- a/cget/cmake/make.cmake +++ b/cget/cmake/make.cmake @@ -58,14 +58,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -143,14 +135,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" diff --git a/cget/cmake/meson.cmake b/cget/cmake/meson.cmake index 8a82a6e..5aa9485 100644 --- a/cget/cmake/meson.cmake +++ b/cget/cmake/meson.cmake @@ -87,14 +87,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -172,14 +164,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" diff --git a/tools/preamble.cmake b/tools/preamble.cmake index b95f76f..f8e1b5a 100644 --- a/tools/preamble.cmake +++ b/tools/preamble.cmake @@ -43,14 +43,6 @@ macro(preamble PREFIX) endforeach() adjust_path(${PREFIX}_SYSTEM_PATH) - set(${PREFIX}_PKG_CONFIG_PATH) - foreach(P ${PREFIX_PATH} ${PREFIX_SYSTEM_PATH}) - foreach(SUFFIX lib lib${${PREFIX}_ADDRESS_MODEL} share) - list(APPEND ${PREFIX}_PKG_CONFIG_PATH ${P}/${SUFFIX}/pkgconfig) - endforeach() - endforeach() - adjust_path(${PREFIX}_PKG_CONFIG_PATH) - get_property_list(${PREFIX}_COMPILE_FLAGS COMPILE_OPTIONS) get_directory_property(${PREFIX}_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) foreach(DIR ${${PREFIX}_INCLUDE_DIRECTORIES}) @@ -128,14 +120,8 @@ macro(preamble PREFIX) set(${PREFIX}_BASE_ENV_COMMAND ${CMAKE_COMMAND} -E env "PATH=${${PREFIX}_SYSTEM_PATH}${PATH_SEP}$ENV{PATH}" - "PKG_CONFIG_PATH=${${PREFIX}_PKG_CONFIG_PATH}" ) - # TODO: Set also PKG_CONFIG_SYSROOT_DIR - if(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE STREQUAL "ONLY") - list(APPEND ${PREFIX}_BASE_ENV_COMMAND "PKG_CONFIG_LIBDIR=${${PREFIX}_PKG_CONFIG_PATH}") - endif() - set(${PREFIX}_ENV_COMMAND ${${PREFIX}_BASE_ENV_COMMAND} "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" From 7a4db1f9162ad221301aabc22b52ab9dbd26d05c Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 26 Jan 2021 23:00:15 +0100 Subject: [PATCH 45/48] isolate packages --- cget/prefix.py | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index 9180253..a27db23 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -134,22 +134,7 @@ def generate_cmake_toolchain( else_ = cmake_else append_ = cmake_append yield set_('CGET_PREFIX', self.prefix) - yield set_('CMAKE_PREFIX_PATH', self.prefix) - if not no_global_include: - yield if_('${CMAKE_VERSION} VERSION_LESS "3.6.0"', - ['include_directories(SYSTEM ${CGET_PREFIX}/include)'], - else_( - set_('CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include'), - set_('CMAKE_C_STANDARD_INCLUDE_DIRECTORIES', '${CGET_PREFIX}/include') - ) - ) if toolchain: yield ['include({})'.format(util.quote(os.path.abspath(toolchain)))] - yield if_('CMAKE_CROSSCOMPILING', - append_('CMAKE_FIND_ROOT_PATH', self.prefix) - ) - yield if_('CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT', - set_('CMAKE_INSTALL_PREFIX', self.prefix) - ) if cxx: yield set_('CMAKE_CXX_COMPILER', cxx) if cc: yield set_('CMAKE_C_COMPILER', cc) if std: @@ -317,13 +302,7 @@ def install_deps( use_build_cache=False, recipe_deps_only=False ): - if pb.requirements: - dependents = self.from_file(pb.requirements, pb.pkg_src.url) - elif src_dir: - dependents = self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) - else: - return - for dependent in dependents: + for dependent in self.get_dependents(pb, src_dir): transient = dependent.test or dependent.build testing = test or test_all installable = not dependent.test or dependent.test == testing @@ -338,6 +317,17 @@ def install_deps( recipe_deps_only=recipe_deps_only ) + def get_dependents(self, pb, src_dir): + if pb.requirements: + dependents = self.from_file(pb.requirements, pb.pkg_src.url) + elif src_dir: + dependents = self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) + else: + return [] + + def get_real_install_path(self, pb): + return os.path.realpath(self.get_package_directory(pb.to_fname(), 'install')) + @returns(six.string_types) @params(pb=PACKAGE_SOURCE_TYPES, test=bool, test_all=bool, update=bool, track=bool) def install( @@ -419,6 +409,11 @@ def install( os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) shutil.copyfile(pb.cmake, target) # Configure and build + defines = list(pb.define or []).append( + "-DCMAKE_PREFIX_PATH=%s" % ";".join( + ['"%s"' % self.get_real_install_path(dep) for dep in self.get_dependents(pb)] + ) + ) builder.configure( src_dir, defines=pb.define, From c58aaa29408e8b7c7023ad7fe0ea9dfe6b4ac16d Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 26 Jan 2021 23:01:55 +0100 Subject: [PATCH 46/48] actually return dependents --- cget/prefix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index a27db23..931fccc 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -319,9 +319,9 @@ def install_deps( def get_dependents(self, pb, src_dir): if pb.requirements: - dependents = self.from_file(pb.requirements, pb.pkg_src.url) + return self.from_file(pb.requirements, pb.pkg_src.url) elif src_dir: - dependents = self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) + return self.from_file(os.path.join(src_dir, 'requirements.txt'), pb.pkg_src.url) else: return [] From aed37d1d4163b162ec8f7b4e3ca4d60ae22ab024 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 26 Jan 2021 23:03:12 +0100 Subject: [PATCH 47/48] pass src dir to get_dependents --- cget/prefix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cget/prefix.py b/cget/prefix.py index 931fccc..797d759 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -411,7 +411,7 @@ def install( # Configure and build defines = list(pb.define or []).append( "-DCMAKE_PREFIX_PATH=%s" % ";".join( - ['"%s"' % self.get_real_install_path(dep) for dep in self.get_dependents(pb)] + ['"%s"' % self.get_real_install_path(dep) for dep in self.get_dependents(pb, src_dir)] ) ) builder.configure( From 9a5780500de20487913ce83e971ee4372f9d35b1 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Tue, 26 Jan 2021 23:30:59 +0100 Subject: [PATCH 48/48] fix defines passign --- cget/prefix.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cget/prefix.py b/cget/prefix.py index 797d759..1e09460 100644 --- a/cget/prefix.py +++ b/cget/prefix.py @@ -409,14 +409,16 @@ def install( os.rename(target, os.path.join(src_dir, builder.cmake_original_file)) shutil.copyfile(pb.cmake, target) # Configure and build - defines = list(pb.define or []).append( - "-DCMAKE_PREFIX_PATH=%s" % ";".join( + defines = list(pb.define or []) + [ + "CMAKE_PREFIX_PATH=%s" % ";".join( ['"%s"' % self.get_real_install_path(dep) for dep in self.get_dependents(pb, src_dir)] ) - ) + ] + print("defines") + print(defines) builder.configure( src_dir, - defines=pb.define, + defines=defines, generator=generator, install_prefix=install_dir, test=test,