From 8d01bed2c15f3a35fb562b79b967c9a322aced48 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Fri, 19 Feb 2016 18:33:26 -0500 Subject: [PATCH 001/105] Add a -mono suffix to the output of llvm-config --version. Also add a new --mono-api-version option which returns a numerical version which is used by mono to detect which version of the mono llvm fork it is compiled with. --- llvm/CMakeLists.txt | 2 ++ llvm/include/llvm/Config/llvm-config.h.cmake | 3 +++ llvm/tools/llvm-config/llvm-config.cpp | 4 +++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 00d01f4ddbbd2..985c60dff94c5 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -23,6 +23,8 @@ if(NOT DEFINED LLVM_VERSION_SUFFIX) set(LLVM_VERSION_SUFFIX) endif() +set(MONO_API_VERSION 1400) + if (NOT PACKAGE_VERSION) set(PACKAGE_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}") diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake index e8a8c20c7c7ad..32968a1255ee1 100644 --- a/llvm/include/llvm/Config/llvm-config.h.cmake +++ b/llvm/include/llvm/Config/llvm-config.h.cmake @@ -112,4 +112,7 @@ /* Define if building LLVM with LLVM_FORCE_USE_OLD_TOOLCHAIN_LIBS */ #cmakedefine LLVM_FORCE_USE_OLD_TOOLCHAIN $(LLVM_FORCE_USE_OLD_TOOLCHAIN) +/* Mono api version */ +#cmakedefine MONO_API_VERSION ${MONO_API_VERSION} + #endif diff --git a/llvm/tools/llvm-config/llvm-config.cpp b/llvm/tools/llvm-config/llvm-config.cpp index 8ed88f33ead46..4f8dc69d14a89 100644 --- a/llvm/tools/llvm-config/llvm-config.cpp +++ b/llvm/tools/llvm-config/llvm-config.cpp @@ -507,7 +507,9 @@ int main(int argc, char **argv) { if (Arg.startswith("-")) { HasAnyOption = true; if (Arg == "--version") { - OS << PACKAGE_VERSION << '\n'; + OS << PACKAGE_VERSION << "-mono" << '\n'; + } else if (Arg == "--mono-api-version") { + OS << MONO_API_VERSION << '\n'; } else if (Arg == "--prefix") { OS << ActivePrefix << '\n'; } else if (Arg == "--bindir") { From d8126bdc6e9389e15118afb8d03c27298b0ffe15 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Fri, 19 Feb 2016 18:38:02 -0500 Subject: [PATCH 002/105] Add a 'mono.this' custom metadata, this is used to mark an 'alloca' whose associated stack slot index will be saved in MachineFunction, and later can be saved in the mono specific EH tables. --- llvm/include/llvm/CodeGen/MachineFunction.h | 6 ++++++ llvm/lib/CodeGen/MachineFunction.cpp | 2 ++ .../lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp | 11 +++++++++++ 3 files changed, 19 insertions(+) diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index c4767a51b0944..6fae62cb161fb 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -390,6 +390,9 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction { /// \pre Fn, Target, MMI, and FunctionNumber are properly set. void init(); + // Stack slot containing the this pointer for mono compiled functions + int MonoThisSlot; + public: struct VariableDbgInfo { const DILocalVariable *Var; @@ -600,6 +603,9 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction { PseudoSourceValueManager &getPSVManager() const { return *PSVManager; } + int getMonoThisSlot() const { return MonoThisSlot; } + void setMonoThisSlot(int Slot) { MonoThisSlot = Slot; } + /// Return the DataLayout attached to the Module associated to this MF. const DataLayout &getDataLayout() const; diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp index fd5ea5cad072c..e38e5f138a104 100644 --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -211,6 +211,8 @@ void MachineFunction::init() { PSVManager = std::make_unique(*(getSubtarget(). getInstrInfo())); + + MonoThisSlot = -1; } MachineFunction::~MachineFunction() { diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index 85c6eca5775e6..2fe0b2228d834 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -184,6 +184,17 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, for (int *CatchObjPtr : Iter->second) *CatchObjPtr = FrameIndex; } + + // + // The mono exception handling code needs to location of the 'this' pointer + // to handle stack traces containing generic shared methods. + // To implement this, it saves the this pointer to an alloca which is marked with + // the 'mono.this' custom metadata. We save the stack slot used by this alloca + // in MachineFunction, so the dwarf exception info emission code can use it to + // compute the reg+offset for it, and save it into the LSDA. + // + if (AI->getMetadata("mono.this")) + MF->setMonoThisSlot(StaticAllocaMap[AI]); } else { // FIXME: Overaligned static allocas should be grouped into // a single dynamic allocation instead of using a separate From 2a1a6cdfb93d99c745872edf48873c23ecfd4c24 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Mon, 1 Aug 2016 15:05:13 -0400 Subject: [PATCH 003/105] Avoid saving/restoring rsp in PreserveAll calling convention code, it seems to confuse the runtime unwinder on linux: https://llvm.org/bugs/show_bug.cgi?id=28757. --- llvm/lib/Target/X86/X86CallingConv.td | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td index 4dd8a6cdd8982..d78fb316cb9e1 100644 --- a/llvm/lib/Target/X86/X86CallingConv.td +++ b/llvm/lib/Target/X86/X86CallingConv.td @@ -1154,11 +1154,11 @@ def CSR_64_CXX_TLS_Darwin_PE : CalleeSavedRegs<(add RBP)>; // CSRs that are handled explicitly via copies. def CSR_64_CXX_TLS_Darwin_ViaCopy : CalleeSavedRegs<(sub CSR_64_TLS_Darwin, RBP)>; -// All GPRs - except r11 +// All GPRs - except r11/rsp def CSR_64_RT_MostRegs : CalleeSavedRegs<(add CSR_64, RAX, RCX, RDX, RSI, RDI, R8, R9, R10)>; -// All registers - except r11 +// All registers - except r11/rsp def CSR_64_RT_AllRegs : CalleeSavedRegs<(add CSR_64_RT_MostRegs, (sequence "XMM%u", 0, 15))>; def CSR_64_RT_AllRegs_AVX : CalleeSavedRegs<(add CSR_64_RT_MostRegs, From d4bb6952b7dea56b21cdd8f1ab8295517b0cdd65 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Fri, 26 Jul 2019 05:35:31 -0400 Subject: [PATCH 004/105] Add a mono calling convention which allows the passing of an extra argument in a platform specific non-abi register. This is used to implement the passing of rgctx/imt arguments in mono. --- llvm/include/llvm-c/Core.h | 1 + llvm/include/llvm/IR/CallingConv.h | 3 +++ .../Target/AArch64/AArch64CallingConvention.h | 6 +++++ .../AArch64/AArch64CallingConvention.td | 25 +++++++++++++++++++ .../Target/AArch64/AArch64ISelLowering.cpp | 7 ++++++ .../Target/AArch64/AArch64RegisterInfo.cpp | 3 +++ llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp | 5 ++++ llvm/lib/Target/ARM/ARMCallingConv.h | 7 +++++- llvm/lib/Target/ARM/ARMCallingConv.td | 21 ++++++++++++++++ llvm/lib/Target/ARM/ARMISelLowering.cpp | 11 ++++++++ llvm/lib/Target/X86/X86CallingConv.td | 14 +++++++++++ 11 files changed, 102 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h index 09d80841fa5d3..a6d58059619e3 100644 --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -221,6 +221,7 @@ typedef enum { LLVMPreserveAllCallConv = 15, LLVMSwiftCallConv = 16, LLVMCXXFASTTLSCallConv = 17, + LLVMMono1CallConv = 20, LLVMX86StdcallCallConv = 64, LLVMX86FastcallCallConv = 65, LLVMARMAPCSCallConv = 66, diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h index fd28542465225..e97806e24c888 100644 --- a/llvm/include/llvm/IR/CallingConv.h +++ b/llvm/include/llvm/IR/CallingConv.h @@ -91,6 +91,9 @@ namespace CallingConv { /// clean up their stack. SwiftTail = 20, + // Mono - Calling convention used by Mono + Mono = 21, + // Target - This is the start of the target-specific calling conventions, // e.g. fastcall and thiscall on X86. FirstTargetCC = 64, diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.h b/llvm/lib/Target/AArch64/AArch64CallingConvention.h index 59939e0684ed2..e8250e1fb2dd8 100644 --- a/llvm/lib/Target/AArch64/AArch64CallingConvention.h +++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.h @@ -46,6 +46,12 @@ bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, bool RetCC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State); +bool CC_AArch64_Mono_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, + ISD::ArgFlagsTy ArgFlags, CCState &State); +bool CC_AArch64_Mono_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + CCState &State); } // namespace llvm #endif diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td index f26151536a586..87749f220bbf4 100644 --- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td +++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td @@ -498,3 +498,28 @@ def CSR_AArch64_SVE_AAPCS_SCS : CalleeSavedRegs<(add CSR_AArch64_SVE_AAPCS, X18)>; def CSR_AArch64_AAPCS_SCS : CalleeSavedRegs<(add CSR_AArch64_AAPCS, X18)>; + +//===----------------------------------------------------------------------===// +// AARCH64 Mono calling conventions +//===----------------------------------------------------------------------===// + +let Entry = 1 in +def CC_AArch64_Mono_DarwinPCS : CallingConv<[ + + // Mono marks the parameter it wants to pass in this non-abi register with + // the 'inreg' attribute. + CCIfInReg>, + + CCDelegateTo +]>; + +let Entry = 1 in +def CC_AArch64_Mono_AAPCS : CallingConv<[ + + // Mono marks the parameter it wants to pass in this non-abi register with + // the 'inreg' attribute. + CCIfInReg>, + + CCDelegateTo +]>; + diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index ac5e51e47ddf1..1cdaf8b10f561 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -5354,6 +5354,13 @@ CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC, case CallingConv::AArch64_VectorCall: case CallingConv::AArch64_SVE_VectorCall: return CC_AArch64_AAPCS; + case CallingConv::Mono: + if (Subtarget->isTargetDarwin()) + return CC_AArch64_Mono_DarwinPCS; + else if (Subtarget->isTargetWindows()) + report_fatal_error("Unsupported calling convention."); + else + return CC_AArch64_Mono_AAPCS; } } diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp index d1b901e58d273..35f97ad5b9e64 100644 --- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp @@ -320,6 +320,9 @@ AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const { markSuperRegs(Reserved, AArch64::WSP); markSuperRegs(Reserved, AArch64::WZR); + if (MF.getFunction().getCallingConv() == CallingConv::Mono) + markSuperRegs(Reserved, AArch64::W15); + if (TFI->hasFP(MF) || TT.isOSDarwin()) markSuperRegs(Reserved, AArch64::W29); diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp index c543d02ff75aa..2ea90c43597f0 100644 --- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp @@ -208,6 +208,11 @@ getReservedRegs(const MachineFunction &MF) const { // Some targets reserve R9. if (STI.isR9Reserved()) markSuperRegs(Reserved, ARM::R9); + if (MF.getFunction().getCallingConv() == CallingConv::Mono) + // FIXME: This is required for some reason, otherwise llvm treats R8 as a callee-saved registers + // even if we exclude it in getCalleeSavedRegs (). Luckily, R8 can still be used for argument + // passing even if it is 'reserved'. + markSuperRegs(Reserved, ARM::R8); // Reserve D16-D31 if the subtarget doesn't support them. if (!STI.hasD32()) { static_assert(ARM::D31 == ARM::D16 + 15, "Register list not consecutive!"); diff --git a/llvm/lib/Target/ARM/ARMCallingConv.h b/llvm/lib/Target/ARM/ARMCallingConv.h index 7c692f03b4405..19eb5316ba2c6 100644 --- a/llvm/lib/Target/ARM/ARMCallingConv.h +++ b/llvm/lib/Target/ARM/ARMCallingConv.h @@ -47,7 +47,12 @@ bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State); - +bool CC_ARM_Mono_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + CCState &State); +bool CC_ARM_Mono_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + CCState &State); } // namespace llvm #endif diff --git a/llvm/lib/Target/ARM/ARMCallingConv.td b/llvm/lib/Target/ARM/ARMCallingConv.td index a6dbe563a4ab7..b7bedca4aab72 100644 --- a/llvm/lib/Target/ARM/ARMCallingConv.td +++ b/llvm/lib/Target/ARM/ARMCallingConv.td @@ -352,4 +352,25 @@ def CSR_GenericInt : CalleeSavedRegs<(add LR, (sequence "R%u", 12, 0))>; // registers. def CSR_FIQ : CalleeSavedRegs<(add LR, R11, (sequence "R%u", 7, 0))>; +//===----------------------------------------------------------------------===// +// ARM Mono calling conventions +//===----------------------------------------------------------------------===// + +let Entry = 1 in +def CC_ARM_Mono_APCS : CallingConv<[ + // Mono marks the parameter it wants to pass in this non-abi register with + // the 'inreg' attribute. + CCIfInReg>, + + CCDelegateTo +]>; + +let Entry = 1 in +def CC_ARM_Mono_AAPCS : CallingConv<[ + // Mono marks the parameter it wants to pass in this non-abi register with + // the 'inreg' attribute. + CCIfInReg>, + + CCDelegateTo +]>; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 1b41427a1cab1..35510c49de2a8 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -2077,6 +2077,8 @@ ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC, return CallingConv::ARM_AAPCS_VFP; else return CallingConv::ARM_AAPCS; + case CallingConv::Mono: + return CallingConv::Mono; } } @@ -2112,6 +2114,15 @@ CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC, return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS); case CallingConv::CFGuard_Check: return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check); + case CallingConv::Mono: + if (Return) { + return CCAssignFnForNode(CallingConv::C, true, isVarArg); + } else { + if (Subtarget->isAAPCS_ABI()) + return CC_ARM_Mono_AAPCS; + else + return CC_ARM_Mono_APCS; + } } } diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td index d78fb316cb9e1..49d6b42b296ce 100644 --- a/llvm/lib/Target/X86/X86CallingConv.td +++ b/llvm/lib/Target/X86/X86CallingConv.td @@ -761,6 +761,12 @@ def CC_X86_64_AnyReg : CallingConv<[ CCCustom<"CC_X86_AnyReg_Error"> ]>; +def CC_X86_64_Mono : CallingConv<[ + CCIfInReg>, + + CCDelegateTo +]>; + //===----------------------------------------------------------------------===// // X86 C Calling Convention //===----------------------------------------------------------------------===// @@ -1064,6 +1070,12 @@ def CC_Intel_OCL_BI : CallingConv<[ CCDelegateTo ]>; +def CC_X86_32_Mono : CallingConv<[ + CCIfInReg>, + + CCDelegateTo +]>; + //===----------------------------------------------------------------------===// // X86 Root Argument Calling Conventions //===----------------------------------------------------------------------===// @@ -1083,6 +1095,7 @@ def CC_X86_32 : CallingConv<[ CCIfCC<"CallingConv::GHC", CCDelegateTo>, CCIfCC<"CallingConv::HiPE", CCDelegateTo>, CCIfCC<"CallingConv::X86_RegCall", CCDelegateTo>, + CCIfCC<"CallingConv::Mono", CCDelegateTo>, // Otherwise, drop to normal X86-32 CC CCDelegateTo @@ -1103,6 +1116,7 @@ def CC_X86_64 : CallingConv<[ CCIfSubtarget<"isTargetWin64()", CCDelegateTo>>, CCIfCC<"CallingConv::X86_RegCall", CCDelegateTo>, CCIfCC<"CallingConv::X86_INTR", CCCustom<"CC_X86_Intr">>, + CCIfCC<"CallingConv::Mono", CCDelegateTo>, // Mingw64 and native Win64 use Win64 CC CCIfSubtarget<"isTargetWin64()", CCDelegateTo>, From a18c5c56a7be3013d388a178c5396286d7ef31d2 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Fri, 19 Feb 2016 18:39:55 -0500 Subject: [PATCH 005/105] Put back the emitDwarfAdvanceFrameAddr () method into MCStreamer which was moved into MCObjectStreamer so it can be used whhen emitting assembly as well. --- llvm/include/llvm/MC/MCObjectStreamer.h | 2 +- llvm/include/llvm/MC/MCStreamer.h | 3 +++ llvm/lib/MC/MCAsmStreamer.cpp | 32 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index 183fd79fb9fc2..28c270b8fe576 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -162,7 +162,7 @@ class MCObjectStreamer : public MCStreamer { unsigned PointerSize) override; void emitDwarfLineEndEntry(MCSection *Section, MCSymbol *LastLabel) override; void emitDwarfAdvanceFrameAddr(const MCSymbol *LastLabel, - const MCSymbol *Label); + const MCSymbol *Label) override; void emitCVLocDirective(unsigned FunctionId, unsigned FileNo, unsigned Line, unsigned Column, bool PrologueEnd, bool IsStmt, StringRef FileName, SMLoc Loc) override; diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 3d6c512bfe73d..d8900f7512791 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -972,6 +972,9 @@ class MCStreamer { ArrayRef> Ranges, codeview::DefRangeFramePointerRelHeader DRHdr); + virtual void emitDwarfAdvanceFrameAddr(const MCSymbol *LastLabel, + const MCSymbol *Label) {} + /// This implements the CodeView '.cv_stringtable' assembler directive. virtual void emitCVStringTableDirective() {} diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index 61ec941f50b82..6d7190ac2b175 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -178,6 +178,8 @@ class MCAsmStreamer final : public MCStreamer { const MCExpr *Value) override; void emitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override; bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override; + void emitDwarfAdvanceFrameAddr(const MCSymbol *LastLabel, + const MCSymbol *Label) override; void emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) override; void BeginCOFFSymbolDef(const MCSymbol *Symbol) override; @@ -700,6 +702,36 @@ void MCAsmStreamer::emitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) { EmitEOL(); } +static const MCExpr *buildSymbolDiff(MCContext &Context, + const MCSymbol *A, + const MCSymbol *B) { + MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None; + const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context); + const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context); + const MCExpr *AddrDelta = + MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context); + return AddrDelta; +} + +static const MCExpr *forceExpAbs(MCStreamer &Streamer, const MCExpr* Expr) { + MCContext &Context = Streamer.getContext(); + assert(!isa(Expr)); + if (Context.getAsmInfo()->hasAggressiveSymbolFolding()) + return Expr; + + MCSymbol *ABS = Context.createTempSymbol(); + Streamer.emitAssignment(ABS, Expr); + return MCSymbolRefExpr::create(ABS, Context); + } + +void MCAsmStreamer::emitDwarfAdvanceFrameAddr(const MCSymbol *LastLabel, + const MCSymbol *Label) { + emitIntValue(dwarf::DW_CFA_advance_loc4, 1); + const MCExpr *AddrDelta = buildSymbolDiff(getContext(), Label, LastLabel); + AddrDelta = forceExpAbs(*this, AddrDelta); + emitValue(AddrDelta, 4); +} + bool MCAsmStreamer::emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) { switch (Attribute) { From 3b5d2380563fef452c649b4dacf3eb1b66b0bc8b Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Fri, 19 Feb 2016 18:42:13 -0500 Subject: [PATCH 006/105] Add support for emitting mono specific EH tables. These tables can be used to map a mono method index to its associated EH and unwind info. --- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 25 +- llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt | 1 + llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp | 2 + llvm/lib/CodeGen/AsmPrinter/MonoException.cpp | 675 ++++++++++++++++++ llvm/lib/CodeGen/AsmPrinter/MonoException.h | 82 +++ 5 files changed, 780 insertions(+), 5 deletions(-) create mode 100644 llvm/lib/CodeGen/AsmPrinter/MonoException.cpp create mode 100644 llvm/lib/CodeGen/AsmPrinter/MonoException.h diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 3e8e190eecc3f..d1009a625c24c 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -18,6 +18,7 @@ #include "WasmException.h" #include "WinCFGuard.h" #include "WinException.h" +#include "MonoException.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" @@ -157,6 +158,12 @@ const char PPGroupDescription[] = "Pseudo Probe Emission"; STATISTIC(EmittedInsts, "Number of machine instrs printed"); +cl::opt EnableMonoEH("enable-mono-eh-frame", cl::NotHidden, + cl::desc("Enable generation of Mono specific EH tables")); + +static cl::opt DisableGNUEH("disable-gnu-eh-frame", cl::NotHidden, + cl::desc("Disable generation of GNU .eh_frame")); + char AsmPrinter::ID = 0; using gcp_map_type = DenseMap>; @@ -396,10 +403,12 @@ bool AsmPrinter::doInitialization(Module &M) { LLVM_FALLTHROUGH; case ExceptionHandling::SjLj: case ExceptionHandling::DwarfCFI: - ES = new DwarfCFIException(this); + if (!DisableGNUEH) + ES = new DwarfCFIException(this); break; case ExceptionHandling::ARM: - ES = new ARMException(this); + if (!DisableGNUEH) + ES = new ARMException(this); break; case ExceptionHandling::WinEH: switch (MAI->getWinEHEncodingType()) { @@ -436,6 +445,11 @@ bool AsmPrinter::doInitialization(Module &M) { HI.Handler->beginModule(&M); } + if (EnableMonoEH) { + MonoException *mono_eh = new MonoException (this, DisableGNUEH); + Handlers.push_back(HandlerInfo(std::unique_ptr (mono_eh), EHTimerName, EHTimerDescription, DWARFGroupName, DWARFGroupDescription)); + } + return false; } @@ -1302,7 +1316,8 @@ void AsmPrinter::emitFunctionBody() { switch (MI.getOpcode()) { case TargetOpcode::CFI_INSTRUCTION: - emitCFIInstruction(MI); + if (!EnableMonoEH) + emitCFIInstruction(MI); break; case TargetOpcode::LOCAL_ESCAPE: emitFrameAlloc(MI); @@ -1477,7 +1492,7 @@ void AsmPrinter::emitFunctionBody() { emitFunctionBodyEnd(); if (needFuncLabelsForEHOrDebugInfo(*MF) || - MAI->hasDotTypeDotSizeDirective()) { + MAI->hasDotTypeDotSizeDirective() || EnableMonoEH) { // Create a symbol for the end of function. CurrentFnEnd = createTempSymbol("func_end"); OutStreamer->emitLabel(CurrentFnEnd); @@ -2035,7 +2050,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { F.hasFnAttribute("function-instrument") || F.hasFnAttribute("xray-instruction-threshold") || needFuncLabelsForEHOrDebugInfo(MF) || NeedsLocalForSize || - MF.getTarget().Options.EmitStackSizeSection || MF.hasBBLabels()) { + MF.getTarget().Options.EmitStackSizeSection || MF.hasBBLabels() || EnableMonoEH) { CurrentFnBegin = createTempSymbol("func_begin"); if (NeedsLocalForSize) CurrentFnSymForSize = CurrentFnBegin; diff --git a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt index eb924282a75e5..ec975555ef1b8 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt +++ b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt @@ -26,6 +26,7 @@ add_llvm_component_library(LLVMAsmPrinter WinException.cpp CodeViewDebug.cpp WasmException.cpp + MonoException.cpp DEPENDS intrinsics_gen diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index 609b568f28beb..6da0269736070 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -1959,6 +1959,8 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU, // Process beginning of an instruction. void DwarfDebug::beginInstruction(const MachineInstr *MI) { + if (!MMI->hasDebugInfo()) + return; const MachineFunction &MF = *MI->getMF(); const auto *SP = MF.getFunction().getSubprogram(); bool NoDebug = diff --git a/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp new file mode 100644 index 0000000000000..c1f29ff9c4633 --- /dev/null +++ b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp @@ -0,0 +1,675 @@ +//===---*- mode: c++; indent-tabs-mode: nil; c-basic-offset: 2 -*---------===// +//===-- CodeGen/AsmPrinter/MonoException.cpp - Dwarf Exception Impl ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains code to emit Mono specific exception handling tables. +// It is based on code in DwarfException.cpp and MCDwarf.cpp. +// +//===----------------------------------------------------------------------===// + +#include "MonoException.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Constants.h" +#include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/CodeGen/TargetFrameLowering.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCSection.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCSymbolELF.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetLoweringObjectFile.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Twine.h" +using namespace llvm; + +// +// The EH tables emitted by this class enable the following functionality: +// - obtaining the GNU EH information for a given method +// - obtaining the DWARF CFI unwind info for a given method +// - obtaining the address of the table itself from user +// code, ie. it is in the text segment pointed to by +// a symbol. +// + +cl::opt MonoEHFrameSymbol("mono-eh-frame-symbol", cl::NotHidden, + cl::desc("Symbol name for the mono eh frame")); + +// Emit a CFI instruction in DWARF format +static void +emitCFIInstruction(MCStreamer &Streamer, + const MCCFIInstruction &Instr, + int &CFAOffset, int DataAlignmentFactor) +{ + // Same as MCDwarf::EmitCFIInstruction () + // FIXME: Unify + int dataAlignmentFactor = DataAlignmentFactor; + bool VerboseAsm = Streamer.isVerboseAsm(); + + switch (Instr.getOperation()) { + case MCCFIInstruction::OpWindowSave: { + Streamer.emitIntValue(dwarf::DW_CFA_GNU_window_save, 1); + return; + } + case MCCFIInstruction::OpUndefined: { + unsigned Reg = Instr.getRegister(); + if (VerboseAsm) { + Streamer.AddComment("DW_CFA_undefined"); + Streamer.AddComment(Twine("Reg ") + Twine(Reg)); + } + Streamer.emitIntValue(dwarf::DW_CFA_undefined, 1); + Streamer.emitULEB128IntValue(Reg); + return; + } + case MCCFIInstruction::OpAdjustCfaOffset: + case MCCFIInstruction::OpDefCfaOffset: { + const bool IsRelative = + Instr.getOperation() == MCCFIInstruction::OpAdjustCfaOffset; + + if (VerboseAsm) + Streamer.AddComment("DW_CFA_def_cfa_offset"); + Streamer.emitIntValue(dwarf::DW_CFA_def_cfa_offset, 1); + + if (IsRelative) { + CFAOffset += Instr.getOffset(); + } else { + // The backends pass in a negative value, + // then createDefCfaOffset () negates it + CFAOffset = Instr.getOffset(); + assert(CFAOffset >= 0); + } + + if (VerboseAsm) + Streamer.AddComment(Twine("Offset " + Twine(CFAOffset))); + Streamer.emitULEB128IntValue(CFAOffset); + + return; + } + case MCCFIInstruction::OpDefCfa: { + if (VerboseAsm) + Streamer.AddComment("DW_CFA_def_cfa"); + Streamer.emitIntValue(dwarf::DW_CFA_def_cfa, 1); + + if (VerboseAsm) + Streamer.AddComment(Twine("Reg ") + Twine(Instr.getRegister())); + Streamer.emitULEB128IntValue(Instr.getRegister()); + + // The backends pass in a negative value, + // then createDefCfaOffset () negates it + CFAOffset = Instr.getOffset(); + if (CFAOffset < 0) { + outs () << CFAOffset << "\n"; + __builtin_trap (); + } + assert(CFAOffset >= 0); + + if (VerboseAsm) + Streamer.AddComment(Twine("Offset " + Twine(CFAOffset))); + Streamer.emitULEB128IntValue(CFAOffset); + + return; + } + + case MCCFIInstruction::OpDefCfaRegister: { + if (VerboseAsm) + Streamer.AddComment("DW_CFA_def_cfa_register"); + Streamer.emitIntValue(dwarf::DW_CFA_def_cfa_register, 1); + + if (VerboseAsm) + Streamer.AddComment(Twine("Reg ") + Twine(Instr.getRegister())); + Streamer.emitULEB128IntValue(Instr.getRegister()); + + return; + } + + case MCCFIInstruction::OpOffset: + case MCCFIInstruction::OpRelOffset: { + const bool IsRelative = + Instr.getOperation() == MCCFIInstruction::OpRelOffset; + + unsigned Reg = Instr.getRegister(); + int Offset = Instr.getOffset(); + if (IsRelative) + Offset -= CFAOffset; + Offset = Offset / dataAlignmentFactor; + + if (Offset < 0) { + if (VerboseAsm) Streamer.AddComment("DW_CFA_offset_extended_sf"); + Streamer.emitIntValue(dwarf::DW_CFA_offset_extended_sf, 1); + if (VerboseAsm) Streamer.AddComment(Twine("Reg ") + Twine(Reg)); + Streamer.emitULEB128IntValue(Reg); + if (VerboseAsm) Streamer.AddComment(Twine("Offset ") + Twine(Offset)); + Streamer.emitSLEB128IntValue(Offset); + } else if (Reg < 64) { + if (VerboseAsm) Streamer.AddComment(Twine("DW_CFA_offset + Reg(") + + Twine(Reg) + ")"); + Streamer.emitIntValue(dwarf::DW_CFA_offset + Reg, 1); + if (VerboseAsm) Streamer.AddComment(Twine("Offset ") + Twine(Offset)); + Streamer.emitULEB128IntValue(Offset); + } else { + if (VerboseAsm) Streamer.AddComment("DW_CFA_offset_extended"); + Streamer.emitIntValue(dwarf::DW_CFA_offset_extended, 1); + if (VerboseAsm) Streamer.AddComment(Twine("Reg ") + Twine(Reg)); + Streamer.emitULEB128IntValue(Reg); + if (VerboseAsm) Streamer.AddComment(Twine("Offset ") + Twine(Offset)); + Streamer.emitULEB128IntValue(Offset); + } + return; + } + case MCCFIInstruction::OpRememberState: + if (VerboseAsm) Streamer.AddComment("DW_CFA_remember_state"); + Streamer.emitIntValue(dwarf::DW_CFA_remember_state, 1); + return; + case MCCFIInstruction::OpRestoreState: + if (VerboseAsm) Streamer.AddComment("DW_CFA_restore_state"); + Streamer.emitIntValue(dwarf::DW_CFA_restore_state, 1); + return; + case MCCFIInstruction::OpSameValue: { + unsigned Reg = Instr.getRegister(); + if (VerboseAsm) Streamer.AddComment("DW_CFA_same_value"); + Streamer.emitIntValue(dwarf::DW_CFA_same_value, 1); + if (VerboseAsm) Streamer.AddComment(Twine("Reg ") + Twine(Reg)); + Streamer.emitULEB128IntValue(Reg); + return; + } + case MCCFIInstruction::OpRestore: { + unsigned Reg = Instr.getRegister(); + if (VerboseAsm) { + Streamer.AddComment("DW_CFA_restore"); + Streamer.AddComment(Twine("Reg ") + Twine(Reg)); + } + Streamer.emitIntValue(dwarf::DW_CFA_restore | Reg, 1); + return; + } + case MCCFIInstruction::OpEscape: + if (VerboseAsm) Streamer.AddComment("Escape bytes"); + Streamer.emitBytes(Instr.getValues()); + return; + case MCCFIInstruction::OpRegister: + case MCCFIInstruction::OpGnuArgsSize: + llvm_unreachable("Unhandled case in switch"); + return; + default: + break; + } + llvm_unreachable("Unhandled case in switch"); +} + +// Emit a list of CFI instructions +static void +emitCFIInstructions(MCStreamer &streamer, + const std::vector &Instrs, + MCSymbol *BaseLabel, + const std::vector *Labels, + int &CFAOffset, + int DataAlignmentFactor) +{ + for (unsigned i = 0, N = Instrs.size(); i < N; ++i) { + const MCCFIInstruction &Instr = Instrs[i]; + MCSymbol *Label = Labels ? ((*Labels)[i]) : NULL; + + // Advance row if new location. + if (BaseLabel && Label) { + MCSymbol *ThisSym = Label; + if (ThisSym != BaseLabel) { + streamer.AddComment ("cfa_advance"); + streamer.emitDwarfAdvanceFrameAddr(BaseLabel, ThisSym); + BaseLabel = ThisSym; + } + } + + emitCFIInstruction(streamer, Instr, CFAOffset, DataAlignmentFactor); + } +} + +MonoException::MonoException(AsmPrinter *A, bool disableGNUEH) + : EHStreamer(A) +{ + RI = nullptr; + DisableGNUEH = disableGNUEH; +} + +MonoException::~MonoException() +{ +} + +void +MonoException::beginFunction(const MachineFunction *MF) +{ + EmitFnStart(); + EHLabels.clear(); +} + +void +MonoException::PrepareMonoLSDA(EHInfo *info) +{ + const MachineFunction *MF = Asm->MF; + const std::vector &TypeInfos = MF->getTypeInfos(); + const std::vector &PadInfos = MF->getLandingPads(); + + // Sort the landing pads in order of their type ids. This is used to fold + // duplicate actions. + SmallVector LandingPads; + LandingPads.reserve(PadInfos.size()); + + for (unsigned i = 0, N = PadInfos.size(); i != N; ++i) + LandingPads.push_back(&PadInfos[i]); + + std::sort(LandingPads.begin(), LandingPads.end(), + [](const LandingPadInfo *L, + const LandingPadInfo *R) { return L->TypeIds < R->TypeIds; }); + + // Invokes and nounwind calls have entries in PadMap (due to being bracketed + // by try-range labels when lowered). Ordinary calls do not, so appropriate + // try-ranges for them need be deduced when using DWARF exception handling. + RangeMapType PadMap; + for (unsigned i = 0, N = LandingPads.size(); i != N; ++i) { + const LandingPadInfo *LandingPad = LandingPads[i]; + for (unsigned j = 0, E = LandingPad->BeginLabels.size(); j != E; ++j) { + MCSymbol *BeginLabel = LandingPad->BeginLabels[j]; + assert(!PadMap.count(BeginLabel) && "Duplicate landing pad labels!"); + PadRange P = { i, j }; + PadMap[BeginLabel] = P; + } + } + + // Compute the call-site table. + SmallVector CallSites; + + MCSymbol *LastLabel = 0; + for (MachineFunction::const_iterator I = MF->begin(), E = MF->end(); + I != E; ++I) { + for (MachineBasicBlock::const_iterator MI = I->begin(), E = I->end(); + MI != E; ++MI) { + if (!MI->isLabel()) { + continue; + } + + MCSymbol *BeginLabel = MI->getOperand(0).getMCSymbol(); + assert(BeginLabel && "Invalid label!"); + + RangeMapType::iterator L = PadMap.find(BeginLabel); + + if (L == PadMap.end()) + continue; + + PadRange P = L->second; + const LandingPadInfo *LandingPad = LandingPads[P.PadIndex]; + + assert(BeginLabel == LandingPad->BeginLabels[P.RangeIndex] && + "Inconsistent landing pad map!"); + + // Mono emits one landing pad for each CLR exception clause, + // and the type info contains the clause index + assert (LandingPad->TypeIds.size() == 1); + assert (LandingPad->LandingPadLabel); + + LastLabel = LandingPad->EndLabels[P.RangeIndex]; + MonoCallSiteEntry Site = {BeginLabel, LastLabel, + LandingPad->LandingPadLabel, LandingPad->TypeIds [0]}; + + assert(Site.BeginLabel && Site.EndLabel && Site.PadLabel && + "Invalid landing pad!"); + + // FIXME: This doesn't work because it includes ranges outside clauses +#if 0 + // Try to merge with the previous call-site. + if (CallSites.size()) { + MonoCallSiteEntry &Prev = CallSites.back(); + if (Site.PadLabel == Prev.PadLabel && Site.TypeID == Prev.TypeID) { + // Extend the range of the previous entry. + Prev.EndLabel = Site.EndLabel; + continue; + } + } +#endif + + // Otherwise, create a new call-site. + CallSites.push_back(Site); + } + } + + info->CallSites.insert(info->CallSites.begin(), CallSites.begin(), CallSites.end()); + info->TypeInfos = TypeInfos; + info->PadInfos = PadInfos; + + int ThisSlot = Asm->MF->getMonoThisSlot(); + + if (ThisSlot != -1) { + llvm::Register FrameReg; + info->ThisOffset = Asm->MF->getTarget ().getSubtargetImpl (Asm->MF->getFunction())->getFrameLowering ()->getFrameIndexReference (*Asm->MF, ThisSlot, FrameReg).getFixed (); + info->FrameReg = Asm->MF->getTarget ().getSubtargetImpl (Asm->MF->getFunction())->getRegisterInfo ()->getDwarfRegNum (FrameReg, true); + } else { + info->FrameReg = -1; + } +} + +void +MonoException::EmitFnStart(void) +{ + if (DisableGNUEH && Asm->MAI->getExceptionHandlingType() == ExceptionHandling::ARM) + static_cast(Asm->OutStreamer->getTargetStreamer())->emitFnStart(); +} + +void +MonoException::EmitFnEnd(void) +{ + if (DisableGNUEH && Asm->MAI->getExceptionHandlingType() == ExceptionHandling::ARM) + static_cast(Asm->OutStreamer->getTargetStreamer())->emitFnEnd(); +} + +void +MonoException::endFunction(const MachineFunction *MF) +{ + // + // Compute a mapping from method names to their AOT method index + // + if (FuncIndexes.size () == 0) { + const Module *m = MMI->getModule (); + NamedMDNode *indexes = m->getNamedMetadata ("mono.function_indexes"); + if (indexes) { + for (unsigned int i = 0; i < indexes->getNumOperands (); ++i) { + MDNode *n = indexes->getOperand (i); + MDString *s = cast(n->getOperand (0)); + auto *idx = mdconst::dyn_extract(n->getOperand (1)); + FuncIndexes.insert (std::make_pair(s->getString (), (int)idx->getLimitedValue () + 1)); + } + } + } + + // Remember the register info + RI = MF->getSubtarget().getRegisterInfo(); + + MachineFunction *NonConstMF = const_cast(MF); + NonConstMF->tidyLandingPads(); + + int monoMethodIdx = FuncIndexes.lookup (Asm->MF->getFunction ().getName ()) - 1; + + if (monoMethodIdx == -1) { + EmitFnEnd (); + return; + } + + //outs () << "D: " << Asm->MF->getFunction()->getName() << " " << monoMethodIdx << "\n"; + + // Save information for use by endModule () + EHInfo info; + + info.FunctionNumber = Asm->getFunctionNumber(); + info.BeginSym = Asm->getFunctionBegin (); + info.EndSym = Asm->getFunctionEnd (); + info.EHLabels = EHLabels; + info.MonoMethodIdx = monoMethodIdx; + info.HasLandingPads = !MF->getLandingPads().empty(); + info.Instructions = MF->getFrameInstructions(); + assert (info.Instructions.size () == info.EHLabels.size()); + + if (DisableGNUEH) + /* ARMAsmPrinter generates references to this */ + Asm->OutStreamer->emitLabel(Asm->getMBBExceptionSym(MF->front())); + + PrepareMonoLSDA(&info); + + Frames.push_back(info); + EHLabels.clear(); + + EmitFnEnd (); +} + +/// EmitMonoLSDA - Mono's version of EmitExceptionTable +/// +/// The code below is a modified/simplified version of DwarfException::EmitExceptionTable() +/// We emit the information inline instead of into a separate section. +/// +void +MonoException::EmitMonoLSDA(const EHInfo *info) +{ + // Load saved information from EHFrameInfo + const std::vector &TypeInfos = info->TypeInfos; + const std::vector &PadInfos = info->PadInfos; + const std::vector CallSites = info->CallSites; + int FrameReg = info->FrameReg; + int ThisOffset = info->ThisOffset; + + // Sort the landing pads in order of their type ids. This is used to fold + // duplicate actions. + SmallVector LandingPads; + LandingPads.reserve(PadInfos.size()); + + for (unsigned i = 0, N = PadInfos.size(); i != N; ++i) + LandingPads.push_back(&PadInfos[i]); + + std::sort(LandingPads.begin(), LandingPads.end(), + [](const LandingPadInfo *L, + const LandingPadInfo *R) { return L->TypeIds < R->TypeIds; }); + + assert(Asm->MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI); + + // The type_info itself is emitted + int TTypeEncoding = dwarf::DW_EH_PE_udata4; + + // Emit the LSDA. + // Keep this in sync with JITDwarfEmitter::EmitExceptionTable () + Asm->emitULEB128(0x4d4fef4f, "MONO Magic"); + Asm->emitULEB128(1, "Version"); + + // Emit the LSDA header. + if (FrameReg != -1) { + Asm->emitEncodingByte(dwarf::DW_EH_PE_udata4, "This encoding"); + + // Emit 'this' location + Asm->OutStreamer->AddComment("bregx"); + Asm->emitInt8((int)dwarf::DW_OP_bregx); + Asm->emitULEB128(FrameReg, "Base reg"); + Asm->emitSLEB128(ThisOffset, "Offset"); + } else { + Asm->emitEncodingByte(dwarf::DW_EH_PE_omit, "@This encoding"); + } + + Asm->emitULEB128 (CallSites.size (), "Number of call sites"); + Asm->emitAlignment(llvm::Align(4)); + for (std::vector::const_iterator + I = CallSites.begin(), E = CallSites.end(); I != E; ++I) { + const MonoCallSiteEntry &S = *I; + + MCSymbol *EHFuncBeginSym = info->BeginSym; + + MCSymbol *BeginLabel = S.BeginLabel; + if (BeginLabel == 0) + BeginLabel = EHFuncBeginSym; + MCSymbol *EndLabel = S.EndLabel; + if (EndLabel == 0) + EndLabel = info->EndSym; + + Asm->OutStreamer->AddComment("Region start"); + Asm->emitLabelDifference(BeginLabel, EHFuncBeginSym, 4); + + Asm->OutStreamer->AddComment("Region length"); + Asm->emitLabelDifference(EndLabel, BeginLabel, 4); + + Asm->OutStreamer->AddComment("Landing pad"); + if (!S.PadLabel) + Asm->OutStreamer->emitIntValue(0, 4); + else + Asm->emitLabelDifference(S.PadLabel, EHFuncBeginSym, 4); + + unsigned int TypeID = S.TypeID; + assert (TypeID > 0 && TypeID <= TypeInfos.size ()); + const GlobalVariable *GV = dyn_cast(TypeInfos[TypeID - 1]); + assert (GV); + + // + // Mono typeinfos are simple constant integers. Emit the constant itself. + // + assert(GV); + const ConstantInt *ci = dyn_cast(GV->getInitializer()); + + Asm->OutStreamer->AddComment("TypeInfo"); + Asm->OutStreamer->emitIntValue(ci->getZExtValue(),Asm->GetSizeOfEncodedValue(TTypeEncoding)); + } +} + +void +MonoException::endModule() +{ + const TargetLoweringObjectFile &tlof = Asm->getObjFileLowering(); + auto &streamer = *Asm->OutStreamer; + + // Size and sign of stack growth. + int stackGrowth = -Asm->getDataLayout().getPointerSize(); + int dataAlignmentFactor = stackGrowth; + + // Emit the EH table + + // Can't use rodata as the symbols we reference are in the text segment + streamer.SwitchSection(tlof.getTextSection()); + + MCSymbol *tableSymbol = + Asm->OutContext.getOrCreateSymbol(Twine(MonoEHFrameSymbol)); + MCSymbol *tableEndSym = Asm->createTempSymbol ("mono_eh_frame_end"); + + // Symbol + Asm->emitAlignment(llvm::Align(16)); + streamer.emitLabel(tableSymbol); + streamer.emitSymbolAttribute(tableSymbol, MCSA_ELF_TypeObject); + if (Asm->MAI->hasDotTypeDotSizeDirective()) { + const MCExpr *SizeExp = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(tableEndSym, Asm->OutContext), + MCSymbolRefExpr::create(tableSymbol, Asm->OutContext), Asm->OutContext); + streamer.emitELFSize(cast(tableSymbol), SizeExp); + } + + // Header + streamer.AddComment("version"); + streamer.emitIntValue(3, 1); + streamer.AddComment ("func addr encoding"); + // Unused + streamer.emitIntValue (0, 1); + + // Search table + Asm->emitAlignment(llvm::Align(4)); + streamer.AddComment("fde_count"); + streamer.emitIntValue (Frames.size(), 4); + + MCSymbol *lastBegin = nullptr; + MCSymbol *lastEnd = nullptr; + for (std::vector::iterator + I = Frames.begin(), E = Frames.end(); I != E; ++I) { + EHInfo &info = *I; + + info.FDESym = Asm->createTempSymbol ("mono_fde"); + + streamer.AddComment("mono method idx"); + streamer.emitIntValue (info.MonoMethodIdx, 4); + + Asm->emitLabelDifference(info.FDESym, tableSymbol, 4); + lastBegin = info.BeginSym; + lastEnd = info.EndSym; + } + + // Emit a last entry to simplify binary searches and to enable the computation of + // the size of the last function/FDE entry + if (Frames.size() == 0) { + streamer.emitIntValue (-1, 4); + Asm->emitLabelDifference(tableSymbol, tableSymbol, 4); + } else { + // Emit the size of the last function, since it cannot be computed using the next table entry + Asm->emitLabelDifference(lastEnd, lastBegin, 4); + Asm->emitLabelDifference(tableEndSym, tableSymbol, 4); + } + + // CIE + // This comes right after the search table + Asm->emitULEB128(1, "CIE Code Alignment Factor"); + Asm->emitSLEB128(stackGrowth, "CIE Data Alignment Factor"); + streamer.AddComment("CIE Return Address Column"); + // RI can be null if there are no methods + if (RI) + Asm->emitInt8(RI->getDwarfRegNum(RI->getRARegister(), true)); + Asm->emitEncodingByte(dwarf::DW_EH_PE_omit, "Personality"); + + int cfaOffset = 0; + + // Initial CIE program + emitCFIInstructions(streamer, streamer.getContext().getAsmInfo()->getInitialFrameState(), NULL, NULL, cfaOffset, stackGrowth); + streamer.AddComment("End of CIE program"); + streamer.emitIntValue(dwarf::DW_CFA_nop, 1); + + int cieCfaOffset = cfaOffset; + + // FDEs + streamer.AddBlankLine(); + for (std::vector::iterator + I = Frames.begin(), E = Frames.end(); I != E; ++I) { + const EHInfo &info = *I; + + streamer.emitLabel(info.FDESym); + + // Emit augmentation + if (info.HasLandingPads || info.FrameReg != -1) { + // Need an extra has_augmentation field as the augmentation size is always encoded + // in 4 bytes + Asm->emitULEB128(1, "Has augmentation"); + + MCSymbol *fdeBeginSym = Asm->OutContext.createTempSymbol("mono_fde_aug_begin", info.FunctionNumber); + MCSymbol *fdeEndSym = Asm->OutContext.createTempSymbol("mono_fde_aug_end", info.FunctionNumber); + + streamer.AddComment("Augmentation size"); + Asm->emitLabelDifference(fdeEndSym, fdeBeginSym, 4); + + streamer.emitLabel(fdeBeginSym); + EmitMonoLSDA (&info); + streamer.emitLabel(fdeEndSym); + } else { + Asm->emitULEB128(0, "Has augmentation"); + } + + // Emit unwind info + cfaOffset = cieCfaOffset; + emitCFIInstructions(streamer, info.Instructions, info.BeginSym, &info.EHLabels, cfaOffset, dataAlignmentFactor); + + streamer.AddBlankLine(); + } + + streamer.emitLabel(tableEndSym); + Asm->emitAlignment(llvm::Align(8)); +} + +void +MonoException::beginInstruction(const MachineInstr *MI) +{ + if (MI->getOpcode() == TargetOpcode::CFI_INSTRUCTION) { + unsigned CFIIndex = MI->getOperand(0).getCFIIndex(); + + //outs () << "D: " << CFIIndex << " " << EHLabels.size() << "\n"; + + /* Emit a label and save the label-cfi index association */ + if (CFIIndex != EHLabels.size()) + assert (0); + + MCSymbol *Label = Asm->OutContext.createTempSymbol(); + Asm->OutStreamer->emitLabel(Label); + + EHLabels.push_back(Label); + } +} diff --git a/llvm/lib/CodeGen/AsmPrinter/MonoException.h b/llvm/lib/CodeGen/AsmPrinter/MonoException.h new file mode 100644 index 0000000000000..625ff398fca54 --- /dev/null +++ b/llvm/lib/CodeGen/AsmPrinter/MonoException.h @@ -0,0 +1,82 @@ +//===---*- mode: c++; indent-tabs-mode: nil -*----------------------------===// +//===-- MonoException.h - Dwarf Exception Framework -----------*- C++ -*--===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// + +#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_MONOEXCEPTION_H +#define LLVM_LIB_CODEGEN_ASMPRINTER_MONOEXCEPTION_H + +#include "EHStreamer.h" +#include "llvm/CodeGen/AsmPrinter.h" + +namespace llvm { + +class TargetRegisterInfo; + +class MonoException : public EHStreamer { +public: + MonoException(AsmPrinter *A, bool disableGNUEH); + virtual ~MonoException(); + + void endModule() override; + + void beginFunction(const MachineFunction *MF) override; + + void endFunction(const MachineFunction *) override; + + void beginInstruction(const MachineInstr *MI) override; +private: + + struct MonoCallSiteEntry { + // The 'try-range' is BeginLabel .. EndLabel. + MCSymbol *BeginLabel; // zero indicates the start of the function. + MCSymbol *EndLabel; // zero indicates the end of the function. + + // The landing pad starts at PadLabel. + MCSymbol *PadLabel; // zero indicates that there is no landing pad. + int TypeID; + }; + + // Per-function EH info + struct EHInfo { + int FunctionNumber, MonoMethodIdx; + MCSymbol *BeginSym, *EndSym, *FDESym; + std::vector EHLabels; + std::vector Instructions; + std::vector CallSites; + std::vector TypeInfos; + std::vector PadInfos; + int FrameReg; + int ThisOffset; + bool HasLandingPads; + + EHInfo() { + FunctionNumber = 0; + MonoMethodIdx = 0; + BeginSym = nullptr; + EndSym = nullptr; + FrameReg = -1; + ThisOffset = 0; + HasLandingPads = 0; + } + }; + + void PrepareMonoLSDA(EHInfo *info); + void EmitMonoLSDA(const EHInfo *info); + void EmitFnStart(); + void EmitFnEnd(); + + std::vector EHLabels; + std::vector Frames; + StringMap FuncIndexes; + const TargetRegisterInfo *RI; + bool DisableGNUEH; +}; +} // End of namespace llvm + +#endif + From 95fc8d9bac0619d1d0ccab799d56ddaf73b47740 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Mon, 30 Apr 2018 14:35:38 -0400 Subject: [PATCH 007/105] Add LLVM_BUILD_EXECUTION_ENGINE and LLVM_TOOLS_TO_BUILD cmake options, they can be used to disable the building of lib/ExecutionEngine and only build a subset of tools. --- llvm/CMakeLists.txt | 9 ++++++++- llvm/cmake/modules/AddLLVM.cmake | 18 ++++++++++++++++-- llvm/cmake/modules/CrossCompile.cmake | 1 + llvm/lib/ExecutionEngine/CMakeLists.txt | 6 +++++- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 985c60dff94c5..bf4b678191ff2 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -610,6 +610,11 @@ if(LLVM_BUILD_EXAMPLES) add_definitions(-DBUILD_EXAMPLES) endif(LLVM_BUILD_EXAMPLES) +option(LLVM_BUILD_EXECUTION_ENGINE + "Build lib/ExecutionEngine." ON) + +set(LLVM_TOOLS_TO_BUILD "all" CACHE STRING "List of tools to build") + option(LLVM_BUILD_TESTS "Build LLVM unit tests. If OFF, just generate build targets." OFF) option(LLVM_INCLUDE_TESTS "Generate build targets for the LLVM unit tests." ON) @@ -1261,5 +1266,7 @@ if (LLVM_INCLUDE_BENCHMARKS) endif() if (LLVM_INCLUDE_UTILS AND LLVM_INCLUDE_TOOLS) - add_subdirectory(utils/llvm-locstats) + if (LLVM_TOOLS_TO_BUILD STREQUAL "all" OR "llvm-locstats" IN_LIST LLVM_TOOLS_TO_BUILD) + add_subdirectory(utils/llvm-locstats) + endif() endif() diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake index 37bc98f9e021e..0d3acacf7d14f 100644 --- a/llvm/cmake/modules/AddLLVM.cmake +++ b/llvm/cmake/modules/AddLLVM.cmake @@ -1374,8 +1374,19 @@ function(add_llvm_subdirectory project type name) option(${canonical_full_name}_BUILD "Whether to build ${name} as part of ${project}" On) mark_as_advanced(${project}_${type}_${name}_BUILD) - if(${canonical_full_name}_BUILD) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir} ${add_llvm_external_dir}) + if("${project}${type}" STREQUAL "LLVMTOOL" AND (NOT "${LLVM_TOOLS_TO_BUILD}" STREQUAL "all")) + if("${name}" IN_LIST LLVM_TOOLS_TO_BUILD) + message("Building tool ${name}.") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir} ${add_llvm_external_dir}) + # Don't process it in add_llvm_implicit_projects(). + set(${project}_${type}_${nameUPPER}_BUILD OFF) + endif() + else() + if(${${project}_${type}_${nameUPPER}_BUILD}) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${add_llvm_external_dir} ${add_llvm_external_dir}) + # Don't process it in add_llvm_implicit_projects(). + set(${project}_${type}_${nameUPPER}_BUILD OFF) + endif() endif() else() set(LLVM_EXTERNAL_${nameUPPER}_SOURCE_DIR @@ -1817,6 +1828,9 @@ function(add_lit_target target comment) message(STATUS "${target} does nothing.") endif() + if( NOT LLVM_BUILD_TESTS ) + return() + endif() if (ARG_DEPENDS) add_dependencies(${target} ${ARG_DEPENDS}) endif() diff --git a/llvm/cmake/modules/CrossCompile.cmake b/llvm/cmake/modules/CrossCompile.cmake index 2a39b6a40a285..557f509c2622d 100644 --- a/llvm/cmake/modules/CrossCompile.cmake +++ b/llvm/cmake/modules/CrossCompile.cmake @@ -84,6 +84,7 @@ function(llvm_create_cross_target project_name target_name toolchain buildtype) -DLLVM_ENABLE_RUNTIMES="${llvm_enable_runtimes_arg}" ${external_project_source_dirs} -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN="${LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN}" + -DLLVM_TOOLS_TO_BUILD="all" ${build_type_flags} ${linker_flag} ${external_clang_dir} ${ARGN} WORKING_DIRECTORY ${${project_name}_${target_name}_BUILD} diff --git a/llvm/lib/ExecutionEngine/CMakeLists.txt b/llvm/lib/ExecutionEngine/CMakeLists.txt index a5607872c7068..d25ad18554826 100644 --- a/llvm/lib/ExecutionEngine/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/CMakeLists.txt @@ -1,4 +1,6 @@ - +if(NOT LLVM_BUILD_EXECUTION_ENGINE) + set(EXCLUDE_FROM_ALL ON) +endif() add_llvm_component_library(LLVMExecutionEngine ExecutionEngine.cpp @@ -27,11 +29,13 @@ if(BUILD_SHARED_LIBS) target_link_libraries(LLVMExecutionEngine PUBLIC LLVMRuntimeDyld) endif() +if(LLVM_BUILD_EXECUTION_ENGINE) add_subdirectory(Interpreter) add_subdirectory(JITLink) add_subdirectory(MCJIT) add_subdirectory(Orc) add_subdirectory(RuntimeDyld) +endif() if( LLVM_USE_OPROFILE ) add_subdirectory(OProfileJIT) From ce23f2fce894262498a61e2143ce0b25b7f4263d Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Mon, 30 Apr 2018 17:08:56 -0400 Subject: [PATCH 008/105] Add a dummy NATIVE.cmake cmake toolchain file which needs to be passed to the NATIVE build of llvm using -DCROSS_TOOLCHAIN_FLAGS_NATIVE=-DCMAKE_TOOLCHAIN_FILE=$PWD/../cmake/modules/NATIVE.cmake when cross compiling, so the native build uses the host compilers, not the cross compilers. --- llvm/cmake/modules/NATIVE.cmake | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 llvm/cmake/modules/NATIVE.cmake diff --git a/llvm/cmake/modules/NATIVE.cmake b/llvm/cmake/modules/NATIVE.cmake new file mode 100644 index 0000000000000..4b0abbfa35085 --- /dev/null +++ b/llvm/cmake/modules/NATIVE.cmake @@ -0,0 +1,2 @@ +set(CMAKE_C_COMPILER cc) +set(CMAKE_CXX_COMPILER c++) From 03dec339906c2470b78f558f33eb9b5ef58bd51f Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 24 Apr 2018 10:44:16 -0400 Subject: [PATCH 009/105] Use LLVM_BUILTIN_TRAP macro, not __builtin_trap(), fixes MSVC --- llvm/lib/CodeGen/AsmPrinter/MonoException.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp index c1f29ff9c4633..cf46cd3ebceb4 100644 --- a/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp @@ -119,7 +119,7 @@ emitCFIInstruction(MCStreamer &Streamer, CFAOffset = Instr.getOffset(); if (CFAOffset < 0) { outs () << CFAOffset << "\n"; - __builtin_trap (); + LLVM_BUILTIN_TRAP; } assert(CFAOffset >= 0); From c4d60900eeb75b1eb69fd760ac1c8a567aa1048c Mon Sep 17 00:00:00 2001 From: Bernhard Urban Date: Wed, 7 Nov 2018 23:10:31 +0100 Subject: [PATCH 010/105] respect hardfloat/softloat setting in ARM ABI (#16) Mono's LLVM backend was always emitting code like this: > 000d7234 : > d7234: e92d4800 push {fp, lr} > d7238: ed2d8b02 vpush {d8} > d723c: ed908b00 vldr d8, [r0] > d7240: eb12e4e2 bl 5905d0 > d7244: e1a03000 mov r3, r0 > d7248: ec510b18 vmov r0, r1, d8 > d724c: e3a02000 mov r2, #0 > d7250: eb12e91a bl 5916c0 > d7254: ecbd8b02 vpop {d8} > d7258: e8bd8800 pop {fp, pc} despite the correct setting passed to `llc`. Note that passing the floating point value (`d8`) in the integer registers `r0` and `r1` is wrong. With this change we get the following: > 000d9934 : > d9934: e92d4800 push {fp, lr} > d9938: ed2d8b02 vpush {d8} > d993c: ed908b00 vldr d8, [r0] > d9940: eb137c2e bl 5b8a00 > d9944: eeb00b48 vmov.f64 d0, d8 > d9948: e1a01000 mov r1, r0 > d994c: e3a00000 mov r0, #0 > d9950: eb138066 bl 5b9af0 > d9954: ecbd8b02 vpop {d8} > d9958: e8bd4800 pop {fp, lr} > d995c: e1a0f00e mov pc, lr Which matches with what the Mono JIT emits: > 0: e92d4100 push {r8, lr} > 4: e24dd028 sub sp, sp, #40 ; 0x28 > 8: e58d0018 str r0, [sp, #24] > c: e59d0018 ldr r0, [sp, #24] > 10: ed900b00 vldr d0, [r0] > 14: ed8d0b08 vstr d0, [sp, #32] > 18: eb00000e bl 0x58 > 1c: e1a01000 mov r1, r0 > 20: ed9d0b08 vldr d0, [sp, #32] > 24: e3a00000 mov r0, #0 > 28: eb000007 bl 0x4c > 2c: e28dd028 add sp, sp, #40 ; 0x28 > 30: e8bd8100 pop {r8, pc} Both are passing the floating point argument correctly via `d0`. Fixes https://github.com/mono/mono/issues/11095 --- llvm/lib/Target/ARM/ARMCallingConv.h | 3 +++ llvm/lib/Target/ARM/ARMCallingConv.td | 9 +++++++++ llvm/lib/Target/ARM/ARMISelLowering.cpp | 10 +++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/ARM/ARMCallingConv.h b/llvm/lib/Target/ARM/ARMCallingConv.h index 19eb5316ba2c6..970cf1bbb6118 100644 --- a/llvm/lib/Target/ARM/ARMCallingConv.h +++ b/llvm/lib/Target/ARM/ARMCallingConv.h @@ -50,6 +50,9 @@ bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, bool CC_ARM_Mono_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State); +bool CC_ARM_Mono_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, + CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, + CCState &State); bool CC_ARM_Mono_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State); diff --git a/llvm/lib/Target/ARM/ARMCallingConv.td b/llvm/lib/Target/ARM/ARMCallingConv.td index b7bedca4aab72..c28cebbf227fb 100644 --- a/llvm/lib/Target/ARM/ARMCallingConv.td +++ b/llvm/lib/Target/ARM/ARMCallingConv.td @@ -374,3 +374,12 @@ def CC_ARM_Mono_AAPCS : CallingConv<[ CCDelegateTo ]>; +let Entry = 1 in +def CC_ARM_Mono_AAPCS_VFP : CallingConv<[ + // Mono marks the parameter it wants to pass in this non-abi register with + // the 'inreg' attribute. + CCIfInReg>, + + CCDelegateTo +]>; + diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 35510c49de2a8..b317bbd5647e5 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -2118,10 +2118,14 @@ CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC, if (Return) { return CCAssignFnForNode(CallingConv::C, true, isVarArg); } else { - if (Subtarget->isAAPCS_ABI()) - return CC_ARM_Mono_AAPCS; - else + if (!Subtarget->isAAPCS_ABI()) return CC_ARM_Mono_APCS; + else if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() && + getTargetMachine().Options.FloatABIType == FloatABI::Hard && + !isVarArg) + return CC_ARM_Mono_AAPCS_VFP; + else + return CC_ARM_Mono_AAPCS; } } } From 5cdcbd4158e5193028c00a438af77cfd7d239eab Mon Sep 17 00:00:00 2001 From: lateralusX Date: Mon, 19 Nov 2018 10:47:50 +0100 Subject: [PATCH 011/105] Fixes Mono LLVM 6.0 for Windows x64. * Adjust mono's LLVM calling convention to use win64 calling convention. * Emit both Windows native unwind info + Dwarf unwind info for mono methods. * Don't emit native personality handlers for mono methods. * Handle Mono and Win64 calling convention similarities when needed. --- llvm/include/llvm/AsmParser/LLToken.h | 1 + llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 5 ++++- llvm/lib/CodeGen/AsmPrinter/MonoException.cpp | 2 -- llvm/lib/CodeGen/AsmPrinter/WinException.cpp | 15 ++++++++++++--- llvm/lib/CodeGen/AsmPrinter/WinException.h | 4 ++++ llvm/lib/IR/AsmWriter.cpp | 1 + llvm/lib/Target/X86/X86CallingConv.td | 3 +++ llvm/lib/Target/X86/X86FastISel.cpp | 6 ++++++ llvm/lib/Target/X86/X86FrameLowering.cpp | 5 ++++- llvm/lib/Target/X86/X86ISelLowering.cpp | 8 +++++--- llvm/lib/Target/X86/X86RegisterInfo.cpp | 6 ++++++ llvm/lib/Target/X86/X86Subtarget.h | 1 + .../Instrumentation/MemorySanitizer.cpp | 12 ++++++++++-- 13 files changed, 57 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 78ebb35e0ea4d..420f82a5f38e7 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -174,6 +174,7 @@ enum Kind { kw_amdgpu_kernel, kw_amdgpu_gfx, kw_tailcc, + kw_monocc, // Attributes: kw_attributes, diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index d1009a625c24c..204d549f452af 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -417,7 +417,10 @@ bool AsmPrinter::doInitialization(Module &M) { break; case WinEH::EncodingType::X86: case WinEH::EncodingType::Itanium: - ES = new WinException(this); + if (!EnableMonoEH) + ES = new WinException(this); + else + ES = new WinException(this, true); break; } break; diff --git a/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp index cf46cd3ebceb4..dc481a494d671 100644 --- a/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp @@ -463,8 +463,6 @@ MonoException::EmitMonoLSDA(const EHInfo *info) [](const LandingPadInfo *L, const LandingPadInfo *R) { return L->TypeIds < R->TypeIds; }); - assert(Asm->MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI); - // The type_info itself is emitted int TTypeEncoding = dwarf::DW_EH_PE_udata4; diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp index ef57031c7294e..4f8ffb84b57c5 100644 --- a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp @@ -46,6 +46,13 @@ WinException::WinException(AsmPrinter *A) : EHStreamer(A) { isThumb = Asm->TM.getTargetTriple().isThumb(); } +WinException::WinException(AsmPrinter *A, bool disableEmitPersonality) +: EHStreamer(A), disableEmitPersonality(disableEmitPersonality) { + // MSVC's EH tables are always composed of 32-bit words. All known 64-bit + // platforms use an imagerel32 relocation to refer to symbols. + useImageRel32 = (A->getDataLayout().getPointerSizeInBits() == 64); +} + WinException::~WinException() {} /// endModule - Emit all exception information that should come after the @@ -91,9 +98,11 @@ void WinException::beginFunction(const MachineFunction *MF) { !isNoOpWithoutInvoke(Per) && F.needsUnwindTableEntry(); - shouldEmitPersonality = - forceEmitPersonality || ((hasLandingPads || hasEHFunclets) && - PerEncoding != dwarf::DW_EH_PE_omit && PerFn); + if (!disableEmitPersonality) { + shouldEmitPersonality = + forceEmitPersonality || ((hasLandingPads || hasEHFunclets) && + PerEncoding != dwarf::DW_EH_PE_omit && PerFn); + } unsigned LSDAEncoding = TLOF.getLSDAEncoding(); shouldEmitLSDA = shouldEmitPersonality && diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.h b/llvm/lib/CodeGen/AsmPrinter/WinException.h index 638589adf0ddc..1deba2016bb5f 100644 --- a/llvm/lib/CodeGen/AsmPrinter/WinException.h +++ b/llvm/lib/CodeGen/AsmPrinter/WinException.h @@ -33,6 +33,9 @@ class LLVM_LIBRARY_VISIBILITY WinException : public EHStreamer { /// Per-function flag to indicate if frame moves info should be emitted. bool shouldEmitMoves = false; + /// Per-function flag to indicate if personality info should be disabled. + bool disableEmitPersonality = false; + /// True if this is a 64-bit target and we should use image relative offsets. bool useImageRel32 = false; @@ -97,6 +100,7 @@ class LLVM_LIBRARY_VISIBILITY WinException : public EHStreamer { // Main entry points. // WinException(AsmPrinter *A); + WinException(AsmPrinter *A, bool disableEmitPersonality); ~WinException() override; /// Emit all exception information that should come after the content. diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 179754e275b03..5bed73a648354 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -336,6 +336,7 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) { case CallingConv::AMDGPU_CS: Out << "amdgpu_cs"; break; case CallingConv::AMDGPU_KERNEL: Out << "amdgpu_kernel"; break; case CallingConv::AMDGPU_Gfx: Out << "amdgpu_gfx"; break; + case CallingConv::Mono: Out << "monocc"; break; } } diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td index 49d6b42b296ce..ca14f4db11f1d 100644 --- a/llvm/lib/Target/X86/X86CallingConv.td +++ b/llvm/lib/Target/X86/X86CallingConv.td @@ -764,6 +764,9 @@ def CC_X86_64_AnyReg : CallingConv<[ def CC_X86_64_Mono : CallingConv<[ CCIfInReg>, + // Mingw64 and native Win64 use Win64 CC + CCIfSubtarget<"isTargetWin64()", CCDelegateTo>, + // Otherwise, drop to normal X86-64 CC CCDelegateTo ]>; diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp index 1ac998b7ff7ed..76a146b6de87d 100644 --- a/llvm/lib/Target/X86/X86FastISel.cpp +++ b/llvm/lib/Target/X86/X86FastISel.cpp @@ -1184,6 +1184,9 @@ bool X86FastISel::X86SelectRet(const Instruction *I) { return false; CallingConv::ID CC = F.getCallingConv(); + if (CC == CallingConv::Mono && Subtarget->isTargetWin64()) + CC = CallingConv::Win64; + if (CC != CallingConv::C && CC != CallingConv::Fast && CC != CallingConv::Tail && @@ -3194,6 +3197,9 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) { if (Subtarget->useIndirectThunkCalls()) return false; + if (CC == CallingConv::Mono && Subtarget->isTargetWin64()) + CC = CallingConv::Win64; + // Handle only C, fastcc, and webkit_js calling conventions for now. switch (CC) { default: return false; diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index 0971a0e33f98b..e73273a548296 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -1410,7 +1410,10 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, bool NeedsWinFPO = !IsFunclet && STI.isTargetWin32() && MMI.getModule()->getCodeViewFlag(); bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO; - bool NeedsDwarfCFI = needsDwarfCFI(MF); + bool NeedsDwarfCFI = + Fn.getCallingConv() == CallingConv::Mono + ? MF.needsFrameMoves() + : needsDwarfCFI(MF); Register FramePtr = TRI->getFrameRegister(MF); const Register MachineFramePtr = STI.isTarget64BitILP32() diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 77c2e7d169907..1fbbd759cd68c 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3423,7 +3423,9 @@ static bool canGuaranteeTCO(CallingConv::ID CC) { } /// Return true if we might ever do TCO for calls with this calling convention. -static bool mayTailCallThisCC(CallingConv::ID CC) { +static bool mayTailCallThisCC(CallingConv::ID CC, bool isTargetWin64) { + if (CC == CallingConv::Mono && isTargetWin64) + CC = CallingConv::Win64; switch (CC) { // C calling conventions: case CallingConv::C: @@ -3454,7 +3456,7 @@ bool X86TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { return false; CallingConv::ID CalleeCC = CI->getCallingConv(); - if (!mayTailCallThisCC(CalleeCC)) + if (!mayTailCallThisCC(CalleeCC, Subtarget.isTargetWin64())) return false; return true; @@ -4861,7 +4863,7 @@ bool X86TargetLowering::IsEligibleForTailCallOptimization( bool isVarArg, Type *RetTy, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, SelectionDAG &DAG) const { - if (!mayTailCallThisCC(CalleeCC)) + if (!mayTailCallThisCC(CalleeCC, Subtarget.isTargetWin64())) return false; // If -tailcallopt is specified, make fastcc functions tail-callable. diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp index 130cb61cdde24..42fd1444163b3 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.cpp +++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp @@ -295,6 +295,9 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { if (MF->getFunction().hasFnAttribute("no_callee_saved_registers")) return CSR_NoRegs_SaveList; + if (CC == CallingConv::Mono && IsWin64) + CC = CallingConv::Win64; + switch (CC) { case CallingConv::GHC: case CallingConv::HiPE: @@ -418,6 +421,9 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF, bool HasAVX = Subtarget.hasAVX(); bool HasAVX512 = Subtarget.hasAVX512(); + if ( CC == CallingConv::Mono && IsWin64) + CC = CallingConv::Win64; + switch (CC) { case CallingConv::GHC: case CallingConv::HiPE: diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h index 5d773f0c57dfb..b19cbbd77b3f7 100644 --- a/llvm/lib/Target/X86/X86Subtarget.h +++ b/llvm/lib/Target/X86/X86Subtarget.h @@ -916,6 +916,7 @@ class X86Subtarget final : public X86GenSubtargetInfo { case CallingConv::X86_ThisCall: case CallingConv::X86_VectorCall: case CallingConv::Intel_OCL_BI: + case CallingConv::Mono: return isTargetWin64(); // This convention allows using the Win64 convention on other targets. case CallingConv::Win64: diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index c51acdf52f14d..addd24ee65882 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -4384,15 +4384,23 @@ struct VarArgAMD64Helper : public VarArgHelper { // nonzero shadow. } + bool isTargetWin64() const { + Triple TargetTriple(F.getParent()->getTargetTriple()); + return TargetTriple.isArch64Bit() && TargetTriple.isOSWindows(); + } + void visitVAStartInst(VAStartInst &I) override { - if (F.getCallingConv() == CallingConv::Win64) + if (F.getCallingConv() == CallingConv::Win64 || + (F.getCallingConv() == CallingConv::Mono && isTargetWin64())) return; VAStartInstrumentationList.push_back(&I); unpoisonVAListTagForInst(I); } void visitVACopyInst(VACopyInst &I) override { - if (F.getCallingConv() == CallingConv::Win64) return; + if (F.getCallingConv() == CallingConv::Win64 || + (F.getCallingConv() == CallingConv::Mono && isTargetWin64())) + return; unpoisonVAListTagForInst(I); } From 13e07411abd9314f9f52c6005f5ee8fea6b7dd7c Mon Sep 17 00:00:00 2001 From: Johan Lorensson Date: Wed, 21 Nov 2018 22:01:38 +0100 Subject: [PATCH 012/105] Workaround for mingw i686 compiler optimization bug. (#19) 32-bit llc.exe build using i686-w64-mingw32-gcc/i686-w64-mingw32-g++ at least on version 5.3.1 and 5.4.0 triggers an optimization bug crashing the process. This was observed by Android team when running cross compiler targeting armv7-linux-gnueabi using LLVM 6.0. The MatchPhiSet call from findCommon triggered an optimization resulting in incorrect stack pointer adjustment on return. That in turn caused an AV when reading the incorrect pointer from stack representing NewPhiNodes stack variable. The fix will reduce optimization around this function not triggering the optimization rules hitting the bug. --- llvm/lib/CodeGen/CodeGenPrepare.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index c888adeafca5d..1f28608f045f8 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -3602,6 +3602,17 @@ class AddressingModeCombiner { // Request is // p -> ? // The function tries to find or build phi [b1, BB1], [b2, BB2] in BB3. + #if defined(__MINGW32__) + // This function triggers an optimization error in i686-w64-mingw32-gcc/i686-w64-mingw32-g++ + // at least observed in 5.3.1 and 5.4.0. Since these mingw releases are the ones currently + // fetched by package managers building LLVM cross compilers targeting Windows, this make sure + // this method only gets compiled with an optimization level not triggering this bug. When the + // optimization error occurs it results in an incorrect stack pointer restore after the call to + // MatchPhiSet and since this method uses FPO next instruction loading NewPhiNodes relative stack pointer + // will get incorrect value triggering an access violation. + #pragma GCC push_options + #pragma GCC optimize ("O1") + #endif Value *findCommon(FoldAddrToValueMapping &Map) { // Tracks the simplification of newly created phi nodes. The reason we use // this mapping is because we will add new created Phi nodes in AddrToBase. @@ -3639,6 +3650,9 @@ class AddressingModeCombiner { } return Result; } + #if defined(__MINGW32__) + #pragma GCC pop_options + #endif /// Try to match PHI node to Candidate. /// Matcher tracks the matched Phi nodes. From 3606e10dffb8d2cce1ca4cb1ccd562d17e4536aa Mon Sep 17 00:00:00 2001 From: lateralusX Date: Mon, 26 Nov 2018 10:06:29 +0100 Subject: [PATCH 013/105] Fix llvm-as to correctly parse monocc. --- llvm/lib/AsmParser/LLLexer.cpp | 1 + llvm/lib/AsmParser/LLParser.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index e3bf41c9721b6..dd7344e32a62f 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -627,6 +627,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(amdgpu_kernel); KEYWORD(amdgpu_gfx); KEYWORD(tailcc); + KEYWORD(monocc); KEYWORD(cc); KEYWORD(c); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 432ec151cf8ae..0c521aebb6191 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -1821,6 +1821,7 @@ void LLParser::parseOptionalDLLStorageClass(unsigned &Res) { /// ::= 'amdgpu_cs' /// ::= 'amdgpu_kernel' /// ::= 'tailcc' +/// ::= 'monocc' /// ::= 'cc' UINT /// bool LLParser::parseOptionalCallingConv(unsigned &CC) { @@ -1873,6 +1874,7 @@ bool LLParser::parseOptionalCallingConv(unsigned &CC) { case lltok::kw_amdgpu_cs: CC = CallingConv::AMDGPU_CS; break; case lltok::kw_amdgpu_kernel: CC = CallingConv::AMDGPU_KERNEL; break; case lltok::kw_tailcc: CC = CallingConv::Tail; break; + case lltok::kw_monocc: CC = CallingConv::Mono; break; case lltok::kw_cc: { Lex.Lex(); return parseUInt32(CC); From be80a1c44975b78688da5215aebe56717cc0ab72 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Wed, 28 Nov 2018 14:39:19 -0800 Subject: [PATCH 014/105] Add Mono GC (#35) --- llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp index a110f7d5c2417..a3138118791b9 100644 --- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp +++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp @@ -450,8 +450,10 @@ static bool shouldRewriteFunction(Function &F) { const auto &FunctionGCName = F.getGC(); const StringRef StatepointExampleName("statepoint-example"); const StringRef CoreCLRName("coreclr"); + const StringRef MonoName("mono"); return (StatepointExampleName == FunctionGCName) || - (CoreCLRName == FunctionGCName); + (CoreCLRName == FunctionGCName) || + (MonoName == FunctionGCName); } else return false; } From 401330ed7602e4027c679add867988439bd1e1eb Mon Sep 17 00:00:00 2001 From: Johan Lorensson Date: Mon, 4 Mar 2019 03:14:18 -0500 Subject: [PATCH 015/105] Merge pull request #39 from lateralusX/lateralusX/fix-dwarf-cfi-windows-x64 Fix correct Dwarf CFI for Windows x64. --- llvm/lib/Target/X86/X86FrameLowering.cpp | 41 +++++++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index e73273a548296..45ed21012de05 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -1519,6 +1519,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, uint64_t NumBytes = 0; int stackGrowth = -SlotSize; + int StackOffset = 2 * stackGrowth; // Find the funclet establisher parameter Register Establisher = X86::NoRegister; @@ -1564,12 +1565,14 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, // Define the current CFA rule to use the provided offset. assert(StackSize); BuildCFI(MBB, MBBI, DL, - MCCFIInstruction::cfiDefCfaOffset(nullptr, -2 * stackGrowth)); + MCCFIInstruction::cfiDefCfaOffset(nullptr, -StackOffset)); // Change the rule for the FramePtr to be an "offset" rule. unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createOffset( - nullptr, DwarfFramePtr, 2 * stackGrowth)); + nullptr, DwarfFramePtr, StackOffset)); + + StackOffset += stackGrowth; } if (NeedsWinCFI) { @@ -1672,7 +1675,6 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, // Skip the callee-saved push instructions. bool PushedRegs = false; - int StackOffset = 2 * stackGrowth; while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup) && @@ -1682,12 +1684,19 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, Register Reg = MBBI->getOperand(0).getReg(); ++MBBI; - if (!HasFP && NeedsDwarfCFI) { + if (!HasFP && NeedsDwarfCFI || (IsWin64Prologue && NeedsDwarfCFI)) { // Mark callee-saved push instruction. // Define the current CFA rule to use the provided offset. assert(StackSize); BuildCFI(MBB, MBBI, DL, MCCFIInstruction::cfiDefCfaOffset(nullptr, -StackOffset)); + + if (IsWin64Prologue && NeedsDwarfCFI) { + unsigned DwarfReg= TRI->getDwarfRegNum(Reg, true); + BuildCFI(MBB, MBBI, DL, + MCCFIInstruction::createOffset(nullptr, DwarfReg, StackOffset)); + } + StackOffset += stackGrowth; } @@ -1794,6 +1803,13 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, .setMIFlag(MachineInstr::FrameSetup); } + // When emitting Dwarf CFI on Win64 account for adjustment at correct + // location in stream. + if (IsWin64Prologue && NeedsDwarfCFI && NumBytes) { + BuildCFI(MBB, MBBI, DL, + MCCFIInstruction::createAdjustCfaOffset(nullptr, NumBytes)); + } + int SEHFrameOffset = 0; unsigned SPOrEstablisher; if (IsFunclet) { @@ -1848,6 +1864,21 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, if (isAsynchronousEHPersonality(Personality)) MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset; } + + if (NeedsDwarfCFI && !IsFunclet) { + // Define the current CFA to use the EBP/RBP register. + unsigned DwarfFramePtr = TRI->getDwarfRegNum(FramePtr, true); + BuildCFI(MBB, MBBI, DL, + MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr)); + + if (SEHFrameOffset) { + // Framepointer has been adjusted with an offset, make sure + // it is reflected in CFA offset. + BuildCFI(MBB, MBBI, DL, + MCCFIInstruction::createAdjustCfaOffset(nullptr, -SEHFrameOffset)); + } + } + } else if (IsFunclet && STI.is32Bit()) { // Reset EBP / ESI to something good for funclets. MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL); @@ -1961,7 +1992,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, } } - if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) { + if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI && !IsWin64Prologue) { // Mark end of stack pointer adjustment. if (!HasFP && NumBytes) { // Define the current CFA rule to use the provided offset. From 0c0c26124b34547516abb71387f0195d5976761f Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Mon, 8 Jul 2019 16:52:35 -0400 Subject: [PATCH 016/105] Merge pull request #48 from lewurm/armv7-hasfp-fix-cfa [arm] fix CFA when frame pointer is used --- llvm/lib/CodeGen/AsmPrinter/MonoException.cpp | 48 +++++++++---------- llvm/lib/CodeGen/AsmPrinter/MonoException.h | 9 +++- llvm/lib/Target/ARM/ARMFrameLowering.cpp | 14 +++++- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp index dc481a494d671..e67964fac4cdf 100644 --- a/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/MonoException.cpp @@ -220,19 +220,22 @@ static void emitCFIInstructions(MCStreamer &streamer, const std::vector &Instrs, MCSymbol *BaseLabel, - const std::vector *Labels, + const std::vector &labeled_cfi, int &CFAOffset, int DataAlignmentFactor) { - for (unsigned i = 0, N = Instrs.size(); i < N; ++i) { - const MCCFIInstruction &Instr = Instrs[i]; - MCSymbol *Label = Labels ? ((*Labels)[i]) : NULL; + for (const auto &lblcfi : labeled_cfi) { + const auto ix = lblcfi.CfiIndex; + const auto Label = lblcfi.Label; + const auto &Instr = Instrs.at(ix); + // Throw out move if the Label is invalid. + if (Label && !Label->isDefined()) continue; // Not emitted, in dead code. // Advance row if new location. if (BaseLabel && Label) { MCSymbol *ThisSym = Label; if (ThisSym != BaseLabel) { - streamer.AddComment ("cfa_advance"); + streamer.AddComment("cfa_advance"); streamer.emitDwarfAdvanceFrameAddr(BaseLabel, ThisSym); BaseLabel = ThisSym; } @@ -257,7 +260,7 @@ void MonoException::beginFunction(const MachineFunction *MF) { EmitFnStart(); - EHLabels.clear(); + CfiInstructions.clear(); } void @@ -418,11 +421,10 @@ MonoException::endFunction(const MachineFunction *MF) info.FunctionNumber = Asm->getFunctionNumber(); info.BeginSym = Asm->getFunctionBegin (); info.EndSym = Asm->getFunctionEnd (); - info.EHLabels = EHLabels; info.MonoMethodIdx = monoMethodIdx; info.HasLandingPads = !MF->getLandingPads().empty(); info.Instructions = MF->getFrameInstructions(); - assert (info.Instructions.size () == info.EHLabels.size()); + info.CfiInstructions = std::move(CfiInstructions); if (DisableGNUEH) /* ARMAsmPrinter generates references to this */ @@ -431,7 +433,6 @@ MonoException::endFunction(const MachineFunction *MF) PrepareMonoLSDA(&info); Frames.push_back(info); - EHLabels.clear(); EmitFnEnd (); } @@ -609,7 +610,12 @@ MonoException::endModule() int cfaOffset = 0; // Initial CIE program - emitCFIInstructions(streamer, streamer.getContext().getAsmInfo()->getInitialFrameState(), NULL, NULL, cfaOffset, stackGrowth); + std::vector cieInstructions; + const unsigned int cieSize = streamer.getContext().getAsmInfo()->getInitialFrameState().size(); + for (unsigned int i = 0; i < cieSize; ++i) { + cieInstructions.push_back({ nullptr, i }); + } + emitCFIInstructions(streamer, streamer.getContext().getAsmInfo()->getInitialFrameState(), nullptr, cieInstructions, cfaOffset, stackGrowth); streamer.AddComment("End of CIE program"); streamer.emitIntValue(dwarf::DW_CFA_nop, 1); @@ -644,7 +650,7 @@ MonoException::endModule() // Emit unwind info cfaOffset = cieCfaOffset; - emitCFIInstructions(streamer, info.Instructions, info.BeginSym, &info.EHLabels, cfaOffset, dataAlignmentFactor); + emitCFIInstructions(streamer, info.Instructions, info.BeginSym, info.CfiInstructions, cfaOffset, dataAlignmentFactor); streamer.AddBlankLine(); } @@ -656,18 +662,10 @@ MonoException::endModule() void MonoException::beginInstruction(const MachineInstr *MI) { - if (MI->getOpcode() == TargetOpcode::CFI_INSTRUCTION) { - unsigned CFIIndex = MI->getOperand(0).getCFIIndex(); - - //outs () << "D: " << CFIIndex << " " << EHLabels.size() << "\n"; - - /* Emit a label and save the label-cfi index association */ - if (CFIIndex != EHLabels.size()) - assert (0); - - MCSymbol *Label = Asm->OutContext.createTempSymbol(); - Asm->OutStreamer->emitLabel(Label); - - EHLabels.push_back(Label); - } + if (MI->getOpcode() == TargetOpcode::CFI_INSTRUCTION) { + const auto cfiIndex = MI->getOperand(0).getCFIIndex(); + const auto label = Asm->OutContext.createTempSymbol(); + Asm->OutStreamer->emitLabel(label); + CfiInstructions.push_back({ label, cfiIndex }); + } } diff --git a/llvm/lib/CodeGen/AsmPrinter/MonoException.h b/llvm/lib/CodeGen/AsmPrinter/MonoException.h index 625ff398fca54..307d2316e15f0 100644 --- a/llvm/lib/CodeGen/AsmPrinter/MonoException.h +++ b/llvm/lib/CodeGen/AsmPrinter/MonoException.h @@ -17,6 +17,11 @@ namespace llvm { class TargetRegisterInfo; +struct MonoLabeledCfiInstruction { + MCSymbol *Label; + unsigned int CfiIndex; +}; + class MonoException : public EHStreamer { public: MonoException(AsmPrinter *A, bool disableGNUEH); @@ -45,11 +50,11 @@ class MonoException : public EHStreamer { struct EHInfo { int FunctionNumber, MonoMethodIdx; MCSymbol *BeginSym, *EndSym, *FDESym; - std::vector EHLabels; std::vector Instructions; std::vector CallSites; std::vector TypeInfos; std::vector PadInfos; + std::vector CfiInstructions; int FrameReg; int ThisOffset; bool HasLandingPads; @@ -70,7 +75,7 @@ class MonoException : public EHStreamer { void EmitFnStart(); void EmitFnEnd(); - std::vector EHLabels; + std::vector CfiInstructions; std::vector Frames; StringMap FuncIndexes; const TargetRegisterInfo *RI; diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp index 1f2f6f7497e02..998b4c3e0088d 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp +++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp @@ -482,6 +482,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, unsigned NumBytes = MFI.getStackSize(); const std::vector &CSI = MFI.getCalleeSavedInfo(); int FPCXTSaveSize = 0; + bool AvoidNonfpCFA = false; // Debug location must be unknown since the first debug location is used // to determine the end of the prologue. @@ -727,6 +728,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, dl, TII, FramePtr, ARM::SP, PushSize + FramePtrOffsetInPush, MachineInstr::FrameSetup); +#if 0 if (FramePtrOffsetInPush + PushSize != 0) { unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa( nullptr, MRI->getDwarfRegNum(FramePtr, true), @@ -742,6 +744,15 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, .addCFIIndex(CFIIndex) .setMIFlags(MachineInstr::FrameSetup); } +#else + /* on iOS `r7 + 8` is always the previous stack pointer */ + unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, MRI->getDwarfRegNum(FramePtr, true), 8)); + BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex) + .setMIFlags(MachineInstr::FrameSetup); + + AvoidNonfpCFA = true; +#endif } // Now that the prologue's actual instructions are finalised, we can insert @@ -831,7 +842,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, // throughout the process. If we have a frame pointer, it takes over the job // half-way through, so only the first few .cfi_def_cfa_offset instructions // actually get emitted. - DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP); + if (!AvoidNonfpCFA) + DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP); if (STI.isTargetELF() && hasFP(MF)) MFI.setOffsetAdjustment(MFI.getOffsetAdjustment() - From 3c78a53aa9b2602bfab501fa3c81fae1d4a1412a Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 25 Oct 2019 14:32:55 -0700 Subject: [PATCH 017/105] Add a flag to disable emission of .llvm_faultmaps metadata. --- llvm/lib/CodeGen/FaultMaps.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/llvm/lib/CodeGen/FaultMaps.cpp b/llvm/lib/CodeGen/FaultMaps.cpp index 1d35b194f2182..fc48277d0f085 100644 --- a/llvm/lib/CodeGen/FaultMaps.cpp +++ b/llvm/lib/CodeGen/FaultMaps.cpp @@ -15,9 +15,16 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/CommandLine.h" using namespace llvm; +static cl::opt DisableFaultMaps("disable-fault-maps", + cl::desc("Disables emission of fault map metadata."), + cl::init(false), cl::Hidden); + #define DEBUG_TYPE "faultmaps" static const int FaultMapVersion = 1; @@ -43,6 +50,8 @@ void FaultMaps::recordFaultingOp(FaultKind FaultTy, } void FaultMaps::serializeToFaultMapSection() { + if (DisableFaultMaps) + return; if (FunctionInfos.empty()) return; From f1f846f3c45025c765f29420858ecaa1738c004f Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Thu, 14 Jan 2021 18:22:18 -0500 Subject: [PATCH 018/105] Fix a crash in the ImplicitNullChecks pass. Re: https://github.com/dotnet/runtime/pull/46817#issuecomment-760542379 --- llvm/lib/CodeGen/ImplicitNullChecks.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/CodeGen/ImplicitNullChecks.cpp b/llvm/lib/CodeGen/ImplicitNullChecks.cpp index fc97938ccd3e9..6010609675861 100644 --- a/llvm/lib/CodeGen/ImplicitNullChecks.cpp +++ b/llvm/lib/CodeGen/ImplicitNullChecks.cpp @@ -353,6 +353,8 @@ ImplicitNullChecks::areMemoryOpsAliased(const MachineInstr &MI, return AR_MayAlias; continue; } + if (MMO2->getValue() == nullptr) + return AR_MayAlias; if (!AA->isNoAlias( MemoryLocation::getAfter(MMO1->getValue(), MMO1->getAAInfo()), MemoryLocation::getAfter(MMO2->getValue(), MMO2->getAAInfo()))) From 5561fc94f0786071e1ad349e552adc7034ff4ec6 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Mon, 22 Mar 2021 08:07:16 -0700 Subject: [PATCH 019/105] RDM and DP detection on arm64 Linux There's no guarantee that `ID_AA64ISAR0_EL1` can be read from user-mode, so add to the existing `/proc/cpuinfo` parser instead. See also: - https://www.kernel.org/doc/html/latest/arm64/elf_hwcaps.html - https://developer.arm.com/documentation/100403/0200/register-descriptions/aarch64-system-registers/id-aa64isar0-el1--aarch64-instruction-set-attribute-register-0--el1 - https://elixir.bootlin.com/linux/v5.11.7/source/arch/arm64/kernel/cpuinfo.c#L58 - https://elixir.bootlin.com/linux/v5.11.7/source/arch/arm64/kernel/cpuinfo.c#L66 --- llvm/lib/Support/Host.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp index f6003b783245e..faf012acce24b 100644 --- a/llvm/lib/Support/Host.cpp +++ b/llvm/lib/Support/Host.cpp @@ -1704,6 +1704,8 @@ bool sys::getHostCPUFeatures(StringMap &Features) { StringRef LLVMFeatureStr = StringSwitch(CPUFeatures[I]) #if defined(__aarch64__) .Case("asimd", "neon") + .Case("asimddp", "dotprod") + .Case("asimdrdm", "rdm") .Case("fp", "fp-armv8") .Case("crc32", "crc") #else From 78e8995034111d50bec8b4fc0129e7a56ca0e7a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Fri, 30 Oct 2020 16:45:02 +0100 Subject: [PATCH 020/105] Fix build with VC19.27 Adds change from https://reviews.llvm.org/D86126 --- llvm/include/llvm/Support/type_traits.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/Support/type_traits.h b/llvm/include/llvm/Support/type_traits.h index 7b7d5d991f3f5..be309d04aa774 100644 --- a/llvm/include/llvm/Support/type_traits.h +++ b/llvm/include/llvm/Support/type_traits.h @@ -176,7 +176,7 @@ class is_trivially_copyable { (has_deleted_copy_assign || has_trivial_copy_assign) && (has_deleted_copy_constructor || has_trivial_copy_constructor); -#ifdef HAVE_STD_IS_TRIVIALLY_COPYABLE +#if 0 // see https://reviews.llvm.org/D86126 static_assert(value == std::is_trivially_copyable::value, "inconsistent behavior between llvm:: and std:: implementation of is_trivially_copyable"); #endif From 233980f98b16089410896a4494501dd7a2724e52 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Fri, 29 Apr 2022 13:01:16 -0400 Subject: [PATCH 021/105] Fix LLVMMono1CallConv. --- llvm/include/llvm-c/Core.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h index a6d58059619e3..2bf2fb72c8009 100644 --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -221,7 +221,7 @@ typedef enum { LLVMPreserveAllCallConv = 15, LLVMSwiftCallConv = 16, LLVMCXXFASTTLSCallConv = 17, - LLVMMono1CallConv = 20, + LLVMMono1CallConv = 21, LLVMX86StdcallCallConv = 64, LLVMX86FastcallCallConv = 65, LLVMARMAPCSCallConv = 66, From a13a2d3a6cb556ddbf99607c393efc253a1c1df0 Mon Sep 17 00:00:00 2001 From: Zoltan Varga Date: Fri, 29 Apr 2022 20:40:54 -0400 Subject: [PATCH 022/105] Sync packaging and building from release/11.x branch (16a14adae6df02f79c9281130b2131329ef95ef2..7f00c63a4c31ae647c7fd0525d1dd2a72afb2e9b) --- .gitignore | 3 - Directory.Build.props | 5 + Directory.Build.targets | 4 + NuGet.config | 16 + azure-pipelines.yml | 177 ++++ build.cmd | 8 + build.sh | 33 + dir.common.props | 80 ++ eng/Build.props | 6 + eng/Publishing.props | 6 + eng/Tools.props | 11 + eng/Version.Details.xml | 19 + eng/Versions.props | 11 + eng/build.ps1 | 132 +++ eng/build.sh | 170 ++++ eng/common-variables.yml | 40 + eng/common/CIBuild.cmd | 2 + eng/common/PSScriptAnalyzerSettings.psd1 | 11 + eng/common/README.md | 28 + eng/common/SetupNugetSources.ps1 | 167 ++++ eng/common/SetupNugetSources.sh | 191 ++++ eng/common/build.ps1 | 161 ++++ eng/common/build.sh | 232 +++++ eng/common/cibuild.sh | 16 + eng/common/cross/arm/sources.list.bionic | 11 + eng/common/cross/arm/sources.list.jessie | 3 + eng/common/cross/arm/sources.list.xenial | 11 + eng/common/cross/arm/sources.list.zesty | 11 + eng/common/cross/arm/tizen-build-rootfs.sh | 35 + eng/common/cross/arm/tizen-fetch.sh | 170 ++++ eng/common/cross/arm/tizen/tizen.patch | 9 + eng/common/cross/arm64/sources.list.bionic | 11 + eng/common/cross/arm64/sources.list.buster | 11 + eng/common/cross/arm64/sources.list.stretch | 12 + eng/common/cross/arm64/sources.list.xenial | 11 + eng/common/cross/arm64/sources.list.zesty | 11 + eng/common/cross/arm64/tizen-build-rootfs.sh | 35 + eng/common/cross/arm64/tizen-fetch.sh | 170 ++++ eng/common/cross/arm64/tizen/tizen.patch | 9 + eng/common/cross/armel/armel.jessie.patch | 43 + eng/common/cross/armel/sources.list.jessie | 3 + eng/common/cross/armel/tizen-build-rootfs.sh | 35 + eng/common/cross/armel/tizen-fetch.sh | 170 ++++ eng/common/cross/armel/tizen/tizen-dotnet.ks | 50 + eng/common/cross/armel/tizen/tizen.patch | 9 + eng/common/cross/armv6/sources.list.buster | 2 + eng/common/cross/build-android-rootfs.sh | 131 +++ eng/common/cross/build-rootfs.sh | 383 ++++++++ eng/common/cross/ppc64le/sources.list.bionic | 11 + eng/common/cross/s390x/sources.list.bionic | 11 + eng/common/cross/toolchain.cmake | 286 ++++++ eng/common/cross/x86/sources.list.bionic | 11 + eng/common/cross/x86/sources.list.xenial | 11 + eng/common/cross/x86/tizen-build-rootfs.sh | 35 + eng/common/cross/x86/tizen-fetch.sh | 170 ++++ eng/common/cross/x86/tizen/tizen.patch | 9 + eng/common/darc-init.ps1 | 47 + eng/common/darc-init.sh | 82 ++ eng/common/dotnet-install.cmd | 2 + eng/common/dotnet-install.ps1 | 28 + eng/common/dotnet-install.sh | 87 ++ eng/common/enable-cross-org-publishing.ps1 | 13 + eng/common/generate-locproject.ps1 | 117 +++ eng/common/generate-sbom-prep.ps1 | 19 + eng/common/generate-sbom-prep.sh | 22 + eng/common/helixpublish.proj | 26 + eng/common/init-tools-native.cmd | 3 + eng/common/init-tools-native.ps1 | 152 +++ eng/common/init-tools-native.sh | 238 +++++ eng/common/internal-feed-operations.ps1 | 132 +++ eng/common/internal-feed-operations.sh | 141 +++ eng/common/internal/Directory.Build.props | 4 + eng/common/internal/NuGet.config | 7 + eng/common/internal/Tools.csproj | 27 + eng/common/msbuild.ps1 | 28 + eng/common/msbuild.sh | 58 ++ eng/common/native/CommonLibrary.psm1 | 400 ++++++++ eng/common/native/common-library.sh | 172 ++++ eng/common/native/init-compiler.sh | 144 +++ eng/common/native/install-cmake-test.sh | 117 +++ eng/common/native/install-cmake.sh | 117 +++ eng/common/native/install-tool.ps1 | 132 +++ eng/common/pipeline-logging-functions.ps1 | 260 +++++ eng/common/pipeline-logging-functions.sh | 206 ++++ .../post-build/add-build-to-channel.ps1 | 48 + .../post-build/check-channel-consistency.ps1 | 40 + eng/common/post-build/nuget-validation.ps1 | 24 + eng/common/post-build/post-build-utils.ps1 | 91 ++ eng/common/post-build/publish-using-darc.ps1 | 54 ++ .../post-build/sourcelink-validation.ps1 | 319 ++++++ eng/common/post-build/symbols-validation.ps1 | 339 +++++++ .../post-build/trigger-subscriptions.ps1 | 64 ++ eng/common/retain-build.ps1 | 45 + eng/common/sdk-task.ps1 | 97 ++ eng/common/sdl/NuGet.config | 13 + eng/common/sdl/configure-sdl-tool.ps1 | 116 +++ eng/common/sdl/execute-all-sdl-tools.ps1 | 165 ++++ eng/common/sdl/extract-artifact-archives.ps1 | 63 ++ eng/common/sdl/extract-artifact-packages.ps1 | 80 ++ eng/common/sdl/init-sdl.ps1 | 55 ++ eng/common/sdl/packages.config | 4 + eng/common/sdl/run-sdl.ps1 | 49 + eng/common/templates/job/execute-sdl.yml | 131 +++ eng/common/templates/job/job.yml | 261 +++++ eng/common/templates/job/onelocbuild.yml | 106 ++ .../templates/job/publish-build-assets.yml | 121 +++ eng/common/templates/job/source-build.yml | 63 ++ .../templates/job/source-index-stage1.yml | 61 ++ eng/common/templates/jobs/codeql-build.yml | 31 + eng/common/templates/jobs/jobs.yml | 97 ++ eng/common/templates/jobs/source-build.yml | 46 + .../templates/post-build/common-variables.yml | 26 + .../templates/post-build/post-build.yml | 277 ++++++ .../post-build/setup-maestro-vars.yml | 70 ++ .../post-build/trigger-subscription.yml | 13 + .../templates/steps/add-build-to-channel.yml | 13 + eng/common/templates/steps/build-reason.yml | 12 + eng/common/templates/steps/execute-codeql.yml | 32 + eng/common/templates/steps/execute-sdl.yml | 68 ++ eng/common/templates/steps/generate-sbom.yml | 44 + eng/common/templates/steps/publish-logs.yml | 23 + eng/common/templates/steps/retain-build.yml | 28 + eng/common/templates/steps/run-on-unix.yml | 7 + eng/common/templates/steps/run-on-windows.yml | 7 + .../steps/run-script-ifequalelse.yml | 33 + eng/common/templates/steps/send-to-helix.yml | 91 ++ eng/common/templates/steps/source-build.yml | 102 ++ eng/common/templates/steps/telemetry-end.yml | 102 ++ .../templates/steps/telemetry-start.yml | 241 +++++ .../templates/variables/sdl-variables.yml | 7 + eng/common/tools.ps1 | 908 ++++++++++++++++++ eng/common/tools.sh | 550 +++++++++++ global.json | 10 + llvm.proj | 140 +++ llvm/CMakeLists.txt | 5 + llvm/build.sh | 1 + llvm/cmake/modules/AddLLVM.cmake | 74 +- llvm/cmake/modules/TableGen.cmake | 6 +- llvm/version.txt.in | 1 + nuget/Directory.Build.props | 207 ++++ nuget/Directory.Build.targets | 71 ++ .../Directory.Build.props | 4 + ...osoft.NETCore.Runtime.Mono.LLVM.Sdk.builds | 4 + ...soft.NETCore.Runtime.Mono.LLVM.Sdk.pkgproj | 8 + ...rosoft.NETCore.Runtime.Mono.LLVM.Sdk.props | 14 + ...rosoft.NETCore.Runtime.Mono.LLVM.Sdk.props | 14 + ...rosoft.NETCore.Runtime.Mono.LLVM.Sdk.props | 15 + .../Directory.Build.props | 4 + ...oft.NETCore.Runtime.Mono.LLVM.Tools.builds | 4 + ...ft.NETCore.Runtime.Mono.LLVM.Tools.pkgproj | 8 + ...soft.NETCore.Runtime.Mono.LLVM.Tools.props | 11 + ...soft.NETCore.Runtime.Mono.LLVM.Tools.props | 11 + ...soft.NETCore.Runtime.Mono.LLVM.Tools.props | 11 + nuget/builds.targets | 31 + nuget/descriptions.json | 22 + nuget/packageIndex.json | 2 + nuget/packages.builds | 38 + nuget/packaging.props | 39 + 158 files changed, 12180 insertions(+), 12 deletions(-) create mode 100644 Directory.Build.props create mode 100644 Directory.Build.targets create mode 100644 NuGet.config create mode 100644 azure-pipelines.yml create mode 100644 build.cmd create mode 100755 build.sh create mode 100644 dir.common.props create mode 100644 eng/Build.props create mode 100644 eng/Publishing.props create mode 100644 eng/Tools.props create mode 100644 eng/Version.Details.xml create mode 100644 eng/Versions.props create mode 100644 eng/build.ps1 create mode 100755 eng/build.sh create mode 100644 eng/common-variables.yml create mode 100644 eng/common/CIBuild.cmd create mode 100644 eng/common/PSScriptAnalyzerSettings.psd1 create mode 100644 eng/common/README.md create mode 100644 eng/common/SetupNugetSources.ps1 create mode 100644 eng/common/SetupNugetSources.sh create mode 100644 eng/common/build.ps1 create mode 100755 eng/common/build.sh create mode 100755 eng/common/cibuild.sh create mode 100644 eng/common/cross/arm/sources.list.bionic create mode 100644 eng/common/cross/arm/sources.list.jessie create mode 100644 eng/common/cross/arm/sources.list.xenial create mode 100644 eng/common/cross/arm/sources.list.zesty create mode 100644 eng/common/cross/arm/tizen-build-rootfs.sh create mode 100644 eng/common/cross/arm/tizen-fetch.sh create mode 100644 eng/common/cross/arm/tizen/tizen.patch create mode 100644 eng/common/cross/arm64/sources.list.bionic create mode 100644 eng/common/cross/arm64/sources.list.buster create mode 100644 eng/common/cross/arm64/sources.list.stretch create mode 100644 eng/common/cross/arm64/sources.list.xenial create mode 100644 eng/common/cross/arm64/sources.list.zesty create mode 100644 eng/common/cross/arm64/tizen-build-rootfs.sh create mode 100644 eng/common/cross/arm64/tizen-fetch.sh create mode 100644 eng/common/cross/arm64/tizen/tizen.patch create mode 100644 eng/common/cross/armel/armel.jessie.patch create mode 100644 eng/common/cross/armel/sources.list.jessie create mode 100755 eng/common/cross/armel/tizen-build-rootfs.sh create mode 100755 eng/common/cross/armel/tizen-fetch.sh create mode 100644 eng/common/cross/armel/tizen/tizen-dotnet.ks create mode 100644 eng/common/cross/armel/tizen/tizen.patch create mode 100644 eng/common/cross/armv6/sources.list.buster create mode 100755 eng/common/cross/build-android-rootfs.sh create mode 100755 eng/common/cross/build-rootfs.sh create mode 100644 eng/common/cross/ppc64le/sources.list.bionic create mode 100644 eng/common/cross/s390x/sources.list.bionic create mode 100644 eng/common/cross/toolchain.cmake create mode 100644 eng/common/cross/x86/sources.list.bionic create mode 100644 eng/common/cross/x86/sources.list.xenial create mode 100644 eng/common/cross/x86/tizen-build-rootfs.sh create mode 100644 eng/common/cross/x86/tizen-fetch.sh create mode 100644 eng/common/cross/x86/tizen/tizen.patch create mode 100644 eng/common/darc-init.ps1 create mode 100755 eng/common/darc-init.sh create mode 100644 eng/common/dotnet-install.cmd create mode 100644 eng/common/dotnet-install.ps1 create mode 100755 eng/common/dotnet-install.sh create mode 100644 eng/common/enable-cross-org-publishing.ps1 create mode 100644 eng/common/generate-locproject.ps1 create mode 100644 eng/common/generate-sbom-prep.ps1 create mode 100644 eng/common/generate-sbom-prep.sh create mode 100644 eng/common/helixpublish.proj create mode 100644 eng/common/init-tools-native.cmd create mode 100644 eng/common/init-tools-native.ps1 create mode 100755 eng/common/init-tools-native.sh create mode 100644 eng/common/internal-feed-operations.ps1 create mode 100755 eng/common/internal-feed-operations.sh create mode 100644 eng/common/internal/Directory.Build.props create mode 100644 eng/common/internal/NuGet.config create mode 100644 eng/common/internal/Tools.csproj create mode 100644 eng/common/msbuild.ps1 create mode 100755 eng/common/msbuild.sh create mode 100644 eng/common/native/CommonLibrary.psm1 create mode 100755 eng/common/native/common-library.sh create mode 100644 eng/common/native/init-compiler.sh create mode 100755 eng/common/native/install-cmake-test.sh create mode 100755 eng/common/native/install-cmake.sh create mode 100644 eng/common/native/install-tool.ps1 create mode 100644 eng/common/pipeline-logging-functions.ps1 create mode 100755 eng/common/pipeline-logging-functions.sh create mode 100644 eng/common/post-build/add-build-to-channel.ps1 create mode 100644 eng/common/post-build/check-channel-consistency.ps1 create mode 100644 eng/common/post-build/nuget-validation.ps1 create mode 100644 eng/common/post-build/post-build-utils.ps1 create mode 100644 eng/common/post-build/publish-using-darc.ps1 create mode 100644 eng/common/post-build/sourcelink-validation.ps1 create mode 100644 eng/common/post-build/symbols-validation.ps1 create mode 100644 eng/common/post-build/trigger-subscriptions.ps1 create mode 100644 eng/common/retain-build.ps1 create mode 100644 eng/common/sdk-task.ps1 create mode 100644 eng/common/sdl/NuGet.config create mode 100644 eng/common/sdl/configure-sdl-tool.ps1 create mode 100644 eng/common/sdl/execute-all-sdl-tools.ps1 create mode 100644 eng/common/sdl/extract-artifact-archives.ps1 create mode 100644 eng/common/sdl/extract-artifact-packages.ps1 create mode 100644 eng/common/sdl/init-sdl.ps1 create mode 100644 eng/common/sdl/packages.config create mode 100644 eng/common/sdl/run-sdl.ps1 create mode 100644 eng/common/templates/job/execute-sdl.yml create mode 100644 eng/common/templates/job/job.yml create mode 100644 eng/common/templates/job/onelocbuild.yml create mode 100644 eng/common/templates/job/publish-build-assets.yml create mode 100644 eng/common/templates/job/source-build.yml create mode 100644 eng/common/templates/job/source-index-stage1.yml create mode 100644 eng/common/templates/jobs/codeql-build.yml create mode 100644 eng/common/templates/jobs/jobs.yml create mode 100644 eng/common/templates/jobs/source-build.yml create mode 100644 eng/common/templates/post-build/common-variables.yml create mode 100644 eng/common/templates/post-build/post-build.yml create mode 100644 eng/common/templates/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates/post-build/trigger-subscription.yml create mode 100644 eng/common/templates/steps/add-build-to-channel.yml create mode 100644 eng/common/templates/steps/build-reason.yml create mode 100644 eng/common/templates/steps/execute-codeql.yml create mode 100644 eng/common/templates/steps/execute-sdl.yml create mode 100644 eng/common/templates/steps/generate-sbom.yml create mode 100644 eng/common/templates/steps/publish-logs.yml create mode 100644 eng/common/templates/steps/retain-build.yml create mode 100644 eng/common/templates/steps/run-on-unix.yml create mode 100644 eng/common/templates/steps/run-on-windows.yml create mode 100644 eng/common/templates/steps/run-script-ifequalelse.yml create mode 100644 eng/common/templates/steps/send-to-helix.yml create mode 100644 eng/common/templates/steps/source-build.yml create mode 100644 eng/common/templates/steps/telemetry-end.yml create mode 100644 eng/common/templates/steps/telemetry-start.yml create mode 100644 eng/common/templates/variables/sdl-variables.yml create mode 100644 eng/common/tools.ps1 create mode 100755 eng/common/tools.sh create mode 100644 global.json create mode 100644 llvm.proj create mode 100644 llvm/build.sh create mode 100644 llvm/version.txt.in create mode 100644 nuget/Directory.Build.props create mode 100644 nuget/Directory.Build.targets create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk/Directory.Build.props create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk.builds create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk.pkgproj create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk/runtime.Linux.Microsoft.NETCore.Runtime.Mono.LLVM.Sdk.props create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk/runtime.OSX.Microsoft.NETCore.Runtime.Mono.LLVM.Sdk.props create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Sdk/runtime.Windows_NT.Microsoft.NETCore.Runtime.Mono.LLVM.Sdk.props create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Tools/Directory.Build.props create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Tools/Microsoft.NETCore.Runtime.Mono.LLVM.Tools.builds create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Tools/Microsoft.NETCore.Runtime.Mono.LLVM.Tools.pkgproj create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Tools/runtime.Linux.Microsoft.NETCore.Runtime.Mono.LLVM.Tools.props create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Tools/runtime.OSX.Microsoft.NETCore.Runtime.Mono.LLVM.Tools.props create mode 100644 nuget/Microsoft.NETCore.Runtime.Mono.LLVM.Tools/runtime.Windows_NT.Microsoft.NETCore.Runtime.Mono.LLVM.Tools.props create mode 100644 nuget/builds.targets create mode 100644 nuget/descriptions.json create mode 100644 nuget/packageIndex.json create mode 100644 nuget/packages.builds create mode 100644 nuget/packaging.props diff --git a/.gitignore b/.gitignore index b33fbbf932379..1e0bf725fa3d1 100644 --- a/.gitignore +++ b/.gitignore @@ -24,9 +24,6 @@ # Ignore the user specified CMake presets in subproject directories. /*/CMakeUserPresets.json -# Nested build directory -/build* - #==============================================================================# # Explicit files to ignore (only matches one). #==============================================================================# diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000000000..3014b766801b9 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,5 @@ + + + + + diff --git a/Directory.Build.targets b/Directory.Build.targets new file mode 100644 index 0000000000000..177d64c47ad11 --- /dev/null +++ b/Directory.Build.targets @@ -0,0 +1,4 @@ + + + + diff --git a/NuGet.config b/NuGet.config new file mode 100644 index 0000000000000..6082e261d277e --- /dev/null +++ b/NuGet.config @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000000..4c031f625b712 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,177 @@ +trigger: + batch: true + branches: + include: + - release/14.x + +pr: + branches: + include: + - release/14.x + +variables: +- template: eng/common-variables.yml + +stages: +- stage: build + displayName: Build + jobs: + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enablePublishBuildArtifacts: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: true + variables: + - _BuildConfig: Release + jobs: + + ############ LINUX BUILD ############ + - job: Build_Linux + displayName: Linux + timeoutInMinutes: 180 + strategy: + matrix: + x64: + assetManifestOS: linux + assetManifestPlatform: x64 + imagename: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-bfcd90a-20200121150017 + rootfs: + archflag: --arch x64 + LLVMTableGenArg: + arm64: + assetManifestOS: linux + assetManifestPlatform: arm64 + imagename: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-cfdd435-20200121150126 + rootfs: /crossrootfs/arm64 + archflag: --arch arm64 + LLVMTableGenArg: /p:LLVMTableGenPath=$(Build.SourcesDirectory)/artifacts/obj/BuildRoot-x64/bin/llvm-tblgen + arm: + assetManifestOS: linux + assetManifestPlatform: arm + imagename: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-09ec757-20200320131433 + rootfs: /crossrootfs/arm + archflag: --arch arm + LLVMTableGenArg: /p:LLVMTableGenPath=$(Build.SourcesDirectory)/artifacts/obj/BuildRoot-x64/bin/llvm-tblgen + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + vmImage: ubuntu-20.04 + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + container: + image: $(imagename) + steps: + - bash: | + set -ex + git clean -ffdx + git reset --hard HEAD + displayName: 'Clean up working directory' + + - bash: | + ./build.sh --ci --restore --build --arch x64 -configuration $(_BuildConfig) $(_InternalBuildArgs) /p:BuildLLVMTableGenOnly=true + displayName: 'Build host llvm-tblgen for cross-compiling' + condition: and(succeeded(), ne(variables['assetManifestPlatform'], 'x64')) + + - bash: | + ./build.sh --ci --restore --build --pack $(archflag) --configuration $(_BuildConfig) $(_InternalBuildArgs) $(LLVMTableGenArg) + displayName: 'Build and package' + env: + ROOTFS_DIR: $(rootfs) + + - bash: | + ./eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) $(_InternalBuildArgs) /p:AssetManifestOS=$(assetManifestOS) /p:PlatformName=$(assetManifestPlatform) --projects $(Build.SourcesDirectory)/llvm.proj + displayName: Publish packages + condition: and(succeeded(), ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) + + ############ MACOS BUILD ############ + - job: Build_macOS + displayName: macOS + timeoutInMinutes: 180 + strategy: + matrix: + x64: + assetManifestOS: osx + assetManifestPlatform: x64 + archflag: --arch x64 + LLVMTableGenArg: + arm64: + assetManifestOS: osx + assetManifestPlatform: arm64 + archflag: --arch arm64 + LLVMTableGenArg: /p:LLVMTableGenPath=$(Build.SourcesDirectory)/artifacts/obj/BuildRoot-x64/bin/llvm-tblgen + pool: + vmImage: macOS-10.15 + steps: + - bash: | + set -ex + git clean -ffdx + git reset --hard HEAD + displayName: 'Clean up working directory' + + - bash: | + ./build.sh --ci --restore --build --arch x64 -configuration $(_BuildConfig) $(_InternalBuildArgs) /p:BuildLLVMTableGenOnly=true + displayName: 'Build host llvm-tblgen for cross-compiling' + condition: and(succeeded(), ne(variables['assetManifestPlatform'], 'x64')) + + - bash: | + ./build.sh --ci --restore --build --pack $(archflag) --configuration $(_BuildConfig) $(_InternalBuildArgs) $(LLVMTableGenArg) + displayName: 'Build and package' + + - bash: + ./eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) $(_InternalBuildArgs) /p:AssetManifestOS=$(assetManifestOS) /p:PlatformName=$(assetManifestPlatform) --projects $(Build.SourcesDirectory)/llvm.proj + displayName: Publish packages + condition: and(succeeded(), ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) + + ############ WINDOWS BUILD ############ + - job: Build_Windows + displayName: Windows + timeoutInMinutes: 180 + strategy: + matrix: + x64: + assetManifestOS: win + assetManifestPlatform: x64 + archflag: -arch x64 + LLVMTableGenArg: + arm64: + assetManifestOS: win + assetManifestPlatform: arm64 + archflag: -arch arm64 + LLVMTableGenArg: /p:LLVMTableGenPath=$(Build.SourcesDirectory)\artifacts\obj\BuildRoot-x64\bin\llvm-tblgen.exe + arm: + assetManifestOS: win + assetManifestPlatform: arm + archflag: -arch arm + LLVMTableGenArg: /p:LLVMTableGenPath=$(Build.SourcesDirectory)\artifacts\obj\BuildRoot-x64\bin\llvm-tblgen.exe + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + vmImage: windows-2019 + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Server.Amd64.VS2019 + steps: + - script: | + git clean -ffdx + git reset --hard HEAD + displayName: 'Clean up working directory' + + - powershell: eng\build.ps1 -ci -restore -build -arch x64 -configuration $(_BuildConfig) $(_InternalBuildArgs) /p:BuildLLVMTableGenOnly=true + displayName: 'Build host llvm-tblgen for cross-compiling' + condition: and(succeeded(), ne(variables['assetManifestPlatform'], 'x64')) + + - powershell: eng\build.ps1 -ci -restore -build -pack $(archflag) -configuration $(_BuildConfig) $(_InternalBuildArgs) $(LLVMTableGenArg) + displayName: 'Build and package' + + - powershell: eng\common\build.ps1 -ci -restore -publish -configuration $(_BuildConfig) $(_InternalBuildArgs) /p:AssetManifestOS=$(assetManifestOS) /p:PlatformName=$(assetManifestPlatform) -projects $(Build.SourcesDirectory)\llvm.proj + displayName: Publish packages + condition: and(succeeded(), ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) + +############ POST BUILD ARCADE LOGIC ############ +- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates/post-build/post-build.yml + parameters: + publishingInfraVersion: 3 + enableSourceLinkValidation: false + enableSigningValidation: false + enableSymbolValidation: false + enableNugetValidation: true diff --git a/build.cmd b/build.cmd new file mode 100644 index 0000000000000..1ba69afd9cc77 --- /dev/null +++ b/build.cmd @@ -0,0 +1,8 @@ +@echo off +setlocal + +set _args=%* +if "%~1"=="-?" set _args=-help + +powershell -ExecutionPolicy ByPass -NoProfile -File "%~dp0eng\build.ps1" %_args% +exit /b %ERRORLEVEL% diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000000..da2ba1a158ec1 --- /dev/null +++ b/build.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +function is_cygwin_or_mingw() +{ + case $(uname -s) in + CYGWIN*) return 0;; + MINGW*) return 0;; + *) return 1;; + esac +} + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +if is_cygwin_or_mingw; then + # if bash shell running on Windows (not WSL), + # pass control to powershell build script. + scriptroot=$(cygpath -d "$scriptroot") + powershell -c "$scriptroot\\build.cmd" $@ +else + "$scriptroot/eng/build.sh" $@ +fi diff --git a/dir.common.props b/dir.common.props new file mode 100644 index 0000000000000..16a9c0c363235 --- /dev/null +++ b/dir.common.props @@ -0,0 +1,80 @@ + + + + + $(__BuildArch) + x64 + x64 + + $(__BuildType) + Debug + Debug + Release + Checked + + $(__BuildOS) + Windows_NT + Linux + OSX + + + <__BuildOS>$(BuildOS) + <__BuildArch>$(BuildArch) + + $(BuildType) + $(BuildArch) + + $(BuildOS).$(BuildArch).$(BuildType) + + + + + + + <__ProjectDir Condition="'$(__ProjectDir)'==''">$(MSBuildThisFileDirectory) + $(__ProjectDir)\ + $(ProjectDir)\..\..\ + $(MSBuildThisFileDirectory) + + $(RootRepoDir)artifacts\obj\llvm\$(MSBuildProjectName)\ + + $(__SourceDir)\ + $(ProjectDir)src\ + + $(__RootBinDir)\ + $(RootRepoDir)artifacts\ + + $(__BinDir)\ + $(RootBinDir)bin\llvm\$(PlatformConfigPathPart)\ + + <_LLVMSourceDir Condition="'$(_LLVMSourceDir)'==''">$(MSBuildThisFileDirectory)/llvm + <_LLVMBuildDir Condition="'$(_LLVMBuildDir)'==''">$(MSBuildThisFileDirectory)/artifacts/obj/BuildRoot-$(TargetArchitecture) + <_LLVMInstallDir Condition="'$(_LLVMInstallDir)'==''">$(MSBuildThisFileDirectory)/artifacts/obj/InstallRoot-$(TargetArchitecture) + + + + + + 1.0.0 + $([System.IO.File]::ReadAllText('$(_LLVMBuildDir)\version.txt')) + + + false + $(PackageVersion) + + preview8 + + + + + true + true + true + true + true + + true + + + + diff --git a/eng/Build.props b/eng/Build.props new file mode 100644 index 0000000000000..a884f3c0863de --- /dev/null +++ b/eng/Build.props @@ -0,0 +1,6 @@ + + + + + + diff --git a/eng/Publishing.props b/eng/Publishing.props new file mode 100644 index 0000000000000..10bc8c8684de3 --- /dev/null +++ b/eng/Publishing.props @@ -0,0 +1,6 @@ + + + + 3 + + diff --git a/eng/Tools.props b/eng/Tools.props new file mode 100644 index 0000000000000..112543dc80edb --- /dev/null +++ b/eng/Tools.props @@ -0,0 +1,11 @@ + + + false + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml new file mode 100644 index 0000000000000..8110e849b8cb4 --- /dev/null +++ b/eng/Version.Details.xml @@ -0,0 +1,19 @@ + + + + + + + https://github.com/dotnet/arcade + bafd55901b50d6fc3507c8ed96a7777fcca1796f + + + https://github.com/dotnet/arcade + bafd55901b50d6fc3507c8ed96a7777fcca1796f + + + https://github.com/dotnet/arcade + bafd55901b50d6fc3507c8ed96a7777fcca1796f + + + diff --git a/eng/Versions.props b/eng/Versions.props new file mode 100644 index 0000000000000..46ff632b565ba --- /dev/null +++ b/eng/Versions.props @@ -0,0 +1,11 @@ + + + + 1.0.0 + alpha + 1 + + + 7.0.0-beta.22168.2 + + diff --git a/eng/build.ps1 b/eng/build.ps1 new file mode 100644 index 0000000000000..bc12cebbdcbd1 --- /dev/null +++ b/eng/build.ps1 @@ -0,0 +1,132 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [switch][Alias('h')]$help, + [switch][Alias('b')]$build, + [switch][Alias('t')]$test, + [switch]$buildtests, + [string][Alias('c')]$configuration = "Debug", + [string][Alias('f')]$framework, + [string]$vs, + [string]$os, + [switch]$allconfigurations, + [switch]$coverage, + [string]$testscope, + [string]$arch, + [string]$runtimeConfiguration, + [string]$librariesConfiguration, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties +) + +function Get-Help() { + Write-Host "Common settings:" + Write-Host " -os Build operating system: Windows_NT or Unix" + Write-Host " -arch Build platform: x86, x64, arm or arm64" + Write-Host " -configuration Build configuration: Debug or Release (short: -c)" + Write-Host " -verbosity MSBuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + Write-Host " -binaryLog Output binary log (short: -bl)" + Write-Host " -help Print help and exit (short: -h)" + Write-Host "" + + Write-Host "Actions (defaults to -restore -build):" + Write-Host " -restore Restore dependencies (short: -r)" + Write-Host " -build Build all source projects (short: -b)" + Write-Host " -buildtests Build all test projects" + Write-Host " -rebuild Rebuild all source projects" + Write-Host " -test Run all unit tests (short: -t)" + Write-Host " -pack Package build outputs into NuGet packages" + Write-Host " -sign Sign build outputs" + Write-Host " -publish Publish artifacts (e.g. symbols)" + Write-Host " -clean Clean the solution" + Write-Host "" + + Write-Host "Libraries settings:" + Write-Host " -vs Open the solution with VS for Test Explorer support. Path or solution name (ie -vs Microsoft.CSharp)" + Write-Host " -framework Build framework: netcoreapp5.0 or net472 (short: -f)" + Write-Host " -coverage Collect code coverage when testing" + Write-Host " -testscope Scope tests, allowed values: innerloop, outerloop, all" + Write-Host " -allconfigurations Build packages for all build configurations" + Write-Host "" + + Write-Host "Command-line arguments not listed above are passed thru to msbuild." + Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -con for configuration, -t for test, etc.)." +} + +if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { + Get-Help + exit 0 +} + +# VS Test Explorer support for libraries +if ($vs) { + . $PSScriptRoot\common\tools.ps1 + + # Microsoft.DotNet.CoreSetup.sln is special - hosting tests are currently meant to run on the + # bootstrapped .NET Core, not on the live-built runtime. + if ([System.IO.Path]::GetFileName($vs) -ieq "Microsoft.DotNet.CoreSetup.sln") { + if (-Not (Test-Path $vs)) { + $vs = Join-Path "$PSScriptRoot\..\src\installer" $vs + } + + # This tells .NET Core to use the bootstrapped runtime to run the tests + $env:DOTNET_ROOT=InitializeDotNetCli -install:$false + } + else { + if (-Not (Test-Path $vs)) { + $vs = Join-Path "$PSScriptRoot\..\src\libraries" $vs | Join-Path -ChildPath "$vs.sln" + } + + $archTestHost = if ($arch) { $arch } else { "x64" } + + # This tells .NET Core to use the same dotnet.exe that build scripts use + $env:DOTNET_ROOT="$PSScriptRoot\..\artifacts\bin\testhost\netcoreapp5.0-Windows_NT-$configuration-$archTestHost"; + $env:DEVPATH="$PSScriptRoot\..\artifacts\bin\testhost\net472-Windows_NT-$configuration-$archTestHost"; + } + + # This tells MSBuild to load the SDK from the directory of the bootstrapped SDK + $env:DOTNET_MSBUILD_SDK_RESOLVER_CLI_DIR=InitializeDotNetCli -install:$false + + # This tells .NET Core not to go looking for .NET Core in other places + $env:DOTNET_MULTILEVEL_LOOKUP=0; + + # Put our local dotnet.exe on PATH first so Visual Studio knows which one to use + $env:PATH=($env:DOTNET_ROOT + ";" + $env:PATH); + + # Launch Visual Studio with the locally defined environment variables + ."$vs" + + exit 0 +} + +# Check if an action is passed in +$actions = "r","restore","b","build","buildtests","rebuild","t","test","pack","sign","publish","clean" +$actionPassedIn = @(Compare-Object -ReferenceObject @($PSBoundParameters.Keys) -DifferenceObject $actions -ExcludeDifferent -IncludeEqual).Length -ne 0 +if ($null -ne $properties -and $actionPassedIn -ne $true) { + $actionPassedIn = @(Compare-Object -ReferenceObject $properties -DifferenceObject $actions.ForEach({ "-" + $_ }) -ExcludeDifferent -IncludeEqual).Length -ne 0 +} + +if (!$actionPassedIn) { + $arguments = "-restore -build" +} + +$possibleDirToBuild = if($properties.Length -gt 0) { $properties[0]; } else { $null } + +foreach ($argument in $PSBoundParameters.Keys) +{ + switch($argument) + { + "build" { $arguments += " -build" } + "buildtests" { if ($build -eq $true) { $arguments += " /p:BuildTests=true" } else { $arguments += " -build /p:BuildTests=only" } } + "test" { $arguments += " -test" } + "configuration" { $configuration = (Get-Culture).TextInfo.ToTitleCase($($PSBoundParameters[$argument])); $arguments += " /p:ConfigurationGroup=$configuration -configuration $configuration" } + "runtimeConfiguration" { $arguments += " /p:RuntimeConfiguration=$((Get-Culture).TextInfo.ToTitleCase($($PSBoundParameters[$argument])))" } + "framework" { $arguments += " /p:BuildTargetFramework=$($PSBoundParameters[$argument].ToLowerInvariant())" } + "os" { $arguments += " /p:OSGroup=$($PSBoundParameters[$argument])" } + "allconfigurations" { $arguments += " /p:BuildAllConfigurations=true" } + "arch" { $arguments += " /p:ArchGroup=$($PSBoundParameters[$argument]) /p:TargetArchitecture=$($PSBoundParameters[$argument])" } + "properties" { $arguments += " " + $properties } + default { $arguments += " /p:$argument=$($PSBoundParameters[$argument])" } + } +} + +Invoke-Expression "& `"$PSScriptRoot/common/build.ps1`" $arguments" +exit $lastExitCode diff --git a/eng/build.sh b/eng/build.sh new file mode 100755 index 0000000000000..becb206742183 --- /dev/null +++ b/eng/build.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash + +set -ue + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +usage() +{ + echo "Common settings:" + echo " --os Build operating system: Windows_NT or Unix" + echo " --arch Build platform: x86, x64, arm or arm64" + echo " --configuration Build configuration: Debug or Release (short: -c)" + echo " --verbosity MSBuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + echo " --binaryLog Output binary log (short: -bl)" + echo " --cross Optional argument to signify cross compilation" + echo " --help Print help and exit (short: -h)" + echo "" + + echo "Actions (defaults to --restore --build):" + echo " --restore Restore dependencies (short: -r)" + echo " --build Build all source projects (short: -b)" + echo " --buildtests Build all test projects" + echo " --rebuild Rebuild all source projects" + echo " --test Run all unit tests (short: -t)" + echo " --pack Package build outputs into NuGet packages" + echo " --sign Sign build outputs" + echo " --publish Publish artifacts (e.g. symbols)" + echo " --clean Clean the solution" + echo "" + + echo "Libraries settings:" + echo " --framework Build framework: netcoreapp or net472 (short: -f)" + echo " --coverage Collect code coverage when testing" + echo " --testscope Test scope, allowed values: innerloop, outerloop, all" + echo " --allconfigurations Build packages for all build configurations" + echo "" + + echo "Native build settings:" + echo " --clang Optional argument to build using clang in PATH (default)" + echo " --clangx.y Optional argument to build using clang version x.y" + echo " --cmakeargs User-settable additional arguments passed to CMake." + echo " --gcc Optional argument to build using gcc in PATH (default)" + echo " --gccx.y Optional argument to build using gcc version x.y" + + echo "Command line arguments starting with '/p:' are passed through to MSBuild." + echo "Arguments can also be passed in with a single hyphen." +} + +arguments='' +cmakeargs='' +extraargs='' +build=false +buildtests=false +checkedPossibleDirectoryToBuild=false + +# Check if an action is passed in +declare -a actions=("r" "restore" "b" "build" "buildtests" "rebuild" "t" "test" "pack" "sign" "publish" "clean") +actInt=($(comm -12 <(printf '%s\n' "${actions[@]/#/-}" | sort) <(printf '%s\n' "${@/#--/-}" | sort))) + +while [[ $# > 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -help|-h) + usage + exit 0 + ;; + -arch) + arguments="$arguments /p:ArchGroup=$2 /p:TargetArchitecture=$2" + shift 2 + ;; + -configuration|-c) + val="$(tr '[:lower:]' '[:upper:]' <<< ${2:0:1})${2:1}" + arguments="$arguments /p:ConfigurationGroup=$val -configuration $val" + shift 2 + ;; + -framework|-f) + val="$(echo "$2" | awk '{print tolower($0)}')" + arguments="$arguments /p:BuildTargetFramework=$val" + shift 2 + ;; + -os) + arguments="$arguments /p:OSGroup=$2" + shift 2 + ;; + -allconfigurations) + arguments="$arguments /p:BuildAllConfigurations=true" + shift 1 + ;; + -build) + build=true + arguments="$arguments -build" + shift 1 + ;; + -buildtests) + buildtests=true + shift 1 + ;; + -testscope) + arguments="$arguments /p:TestScope=$2" + shift 2 + ;; + -coverage) + arguments="$arguments /p:Coverage=true" + shift 1 + ;; + -stripsymbols) + arguments="$arguments /p:BuildNativeStripSymbols=true" + shift 1 + ;; + -runtimeconfiguration) + val="$(tr '[:lower:]' '[:upper:]' <<< ${2:0:1})${2:1}" + arguments="$arguments /p:RuntimeConfiguration=$val" + shift 2 + ;; + -librariesconfiguration) + arguments="$arguments /p:LibrariesConfiguration=$2" + shift 2 + ;; + -cross) + arguments="$arguments /p:CrossBuild=True" + shift 1 + ;; + -clang*) + arguments="$arguments /p:Compiler=$opt" + shift 1 + ;; + -cmakeargs) + cmakeargs="${cmakeargs} ${opt} $2" + shift 2 + ;; + -gcc*) + arguments="$arguments /p:Compiler=$opt" + shift 1 + ;; + *) + ea=$1 + + extraargs="$extraargs $ea" + shift 1 + ;; + esac +done + +if [[ "$buildtests" == true ]]; then + if [[ "$build" == true ]]; then + arguments="$arguments /p:BuildTests=true" + else + arguments="$arguments -build /p:BuildTests=only" + fi +fi + +if [ ${#actInt[@]} -eq 0 ]; then + arguments="-restore -build $arguments" +fi + +# URL-encode space (%20) to avoid quoting issues until the msbuild call in /eng/common/tools.sh. +# In *proj files (XML docs), URL-encoded string are rendered in their decoded form. +cmakeargs="${cmakeargs// /%20}" +arguments="$arguments /p:CMakeArgs=\"$cmakeargs\" $extraargs" +"$scriptroot/common/build.sh" $arguments diff --git a/eng/common-variables.yml b/eng/common-variables.yml new file mode 100644 index 0000000000000..1901c728e063e --- /dev/null +++ b/eng/common-variables.yml @@ -0,0 +1,40 @@ + +variables: + # Cannot use key:value syntax in root defined variables + - name: _TeamName + value: DotNetCore + - name: _DotNetArtifactsCategory + value: .NETCore + - name: _DotNetValidationArtifactsCategory + value: .NETCore + - name: HelixApiAccessToken + value: '' + - name: _RunAsPublic + value: True + - name: _RunAsInternal + value: False + - name: _InternalBuildArgs + value: '' + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: _RunAsPublic + value: False + - name: _RunAsInternal + value: True + - name: _SignType + value: real + # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1 + # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT + # DotNet-HelixApi-Access provides: HelixApiAccessToken + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: DotNet-VSTS-Bot + - group: DotNet-HelixApi-Access + - name: _InternalBuildArgs + value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) + /p:DotNetPublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /p:DotNetPublishToBlobFeed=true + /p:DotNetPublishUsingPipelines=true + /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd new file mode 100644 index 0000000000000..56c2f25ac22ff --- /dev/null +++ b/eng/common/CIBuild.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" \ No newline at end of file diff --git a/eng/common/PSScriptAnalyzerSettings.psd1 b/eng/common/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 0000000000000..4c1ea7c98ea4d --- /dev/null +++ b/eng/common/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,11 @@ +@{ + IncludeRules=@('PSAvoidUsingCmdletAliases', + 'PSAvoidUsingWMICmdlet', + 'PSAvoidUsingPositionalParameters', + 'PSAvoidUsingInvokeExpression', + 'PSUseDeclaredVarsMoreThanAssignments', + 'PSUseCmdletCorrectly', + 'PSStandardDSCFunctionsInResource', + 'PSUseIdenticalMandatoryParametersForDSC', + 'PSUseIdenticalParametersForDSC') +} \ No newline at end of file diff --git a/eng/common/README.md b/eng/common/README.md new file mode 100644 index 0000000000000..ff49c371527a0 --- /dev/null +++ b/eng/common/README.md @@ -0,0 +1,28 @@ +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 new file mode 100644 index 0000000000000..18823840b1127 --- /dev/null +++ b/eng/common/SetupNugetSources.ps1 @@ -0,0 +1,167 @@ +# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. +# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# +# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry +# under for each Maestro managed private feed. Two additional credential +# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. +# +# This script needs to be called in every job that will restore packages and which the base repo has +# private AzDO feeds in the NuGet.config. +# +# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` +# from the AzureDevOps-Artifact-Feeds-Pats variable group. +# +# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing +# +# - task: PowerShell@2 +# displayName: Setup Private Feeds Credentials +# condition: eq(variables['Agent.OS'], 'Windows_NT') +# inputs: +# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 +# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token +# env: +# Token: $(dn-bot-dnceng-artifact-feeds-rw) + +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)][string]$ConfigFile, + [Parameter(Mandatory = $true)][string]$Password +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +. $PSScriptRoot\tools.ps1 + +# Add source entry to PackageSources +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { + $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") + + if ($packageSource -eq $null) + { + $packageSource = $doc.CreateElement("add") + $packageSource.SetAttribute("key", $SourceName) + $packageSource.SetAttribute("value", $SourceEndPoint) + $sources.AppendChild($packageSource) | Out-Null + } + else { + Write-Host "Package source $SourceName already present." + } + + AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password +} + +# Add a credential node for the specified source +function AddCredential($creds, $source, $username, $password) { + # Looks for credential configuration for the given SourceName. Create it if none is found. + $sourceElement = $creds.SelectSingleNode($Source) + if ($sourceElement -eq $null) + { + $sourceElement = $doc.CreateElement($Source) + $creds.AppendChild($sourceElement) | Out-Null + } + + # Add the node to the credential if none is found. + $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']") + if ($usernameElement -eq $null) + { + $usernameElement = $doc.CreateElement("add") + $usernameElement.SetAttribute("key", "Username") + $sourceElement.AppendChild($usernameElement) | Out-Null + } + $usernameElement.SetAttribute("value", $Username) + + # Add the to the credential if none is found. + # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs. + # -> https://github.com/NuGet/Home/issues/5526 + $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']") + if ($passwordElement -eq $null) + { + $passwordElement = $doc.CreateElement("add") + $passwordElement.SetAttribute("key", "ClearTextPassword") + $sourceElement.AppendChild($passwordElement) | Out-Null + } + $passwordElement.SetAttribute("value", $Password) +} + +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { + $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") + + Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." + + ForEach ($PackageSource in $maestroPrivateSources) { + Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + } +} + +function EnablePrivatePackageSources($DisabledPackageSources) { + $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroPrivateSources) { + Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" + # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries + $DisabledPackageSources.RemoveChild($DisabledPackageSource) + } +} + +if (!(Test-Path $ConfigFile -PathType Leaf)) { + Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" + ExitWithExitCode 1 +} + +if (!$Password) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT' + ExitWithExitCode 1 +} + +# Load NuGet.config +$doc = New-Object System.Xml.XmlDocument +$filename = (Get-Item $ConfigFile).FullName +$doc.Load($filename) + +# Get reference to or create one if none exist already +$sources = $doc.DocumentElement.SelectSingleNode("packageSources") +if ($sources -eq $null) { + $sources = $doc.CreateElement("packageSources") + $doc.DocumentElement.AppendChild($sources) | Out-Null +} + +# Looks for a node. Create it if none is found. +$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") +if ($creds -eq $null) { + $creds = $doc.CreateElement("packageSourceCredentials") + $doc.DocumentElement.AppendChild($creds) | Out-Null +} + +# Check for disabledPackageSources; we'll enable any darc-int ones we find there +$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") +if ($disabledSources -ne $null) { + Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" + EnablePrivatePackageSources -DisabledPackageSources $disabledSources +} + +$userName = "dn-bot" + +# Insert credential nodes for Maestro's private feeds +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password + +$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") +if ($dotnet31Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + +$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") +if ($dotnet5Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + +$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") +if ($dotnet6Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + +$doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh new file mode 100644 index 0000000000000..ad3fb74fd2cc8 --- /dev/null +++ b/eng/common/SetupNugetSources.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. +# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# +# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry +# under for each Maestro's managed private feed. Two additional credential +# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. +# +# This script needs to be called in every job that will restore packages and which the base repo has +# private AzDO feeds in the NuGet.config. +# +# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` +# from the AzureDevOps-Artifact-Feeds-Pats variable group. +# +# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing. +# +# - task: Bash@3 +# displayName: Setup Private Feeds Credentials +# inputs: +# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh +# arguments: $(Build.SourcesDirectory)/NuGet.config $Token +# condition: ne(variables['Agent.OS'], 'Windows_NT') +# env: +# Token: $(dn-bot-dnceng-artifact-feeds-rw) + +ConfigFile=$1 +CredToken=$2 +NL='\n' +TB=' ' + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ ! -f "$ConfigFile" ]; then + Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" + ExitWithExitCode 1 +fi + +if [ -z "$CredToken" ]; then + Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT" + ExitWithExitCode 1 +fi + +if [[ `uname -s` == "Darwin" ]]; then + NL=$'\\\n' + TB='' +fi + +# Ensure there is a ... section. +grep -i "" $ConfigFile +if [ "$?" != "0" ]; then + echo "Adding ... section." + ConfigNodeHeader="" + PackageSourcesTemplate="${TB}${NL}${TB}" + + sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile +fi + +# Ensure there is a ... section. +grep -i "" $ConfigFile +if [ "$?" != "0" ]; then + echo "Adding ... section." + + PackageSourcesNodeFooter="" + PackageSourceCredentialsTemplate="${TB}${NL}${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile +fi + +PackageSources=() + +# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present +grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3.1-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet3.1-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3.1-internal-transport') +fi + +# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present +grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet5-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet5-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet5-internal-transport') +fi + +# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present +grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet6-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal-transport') +fi + +# I want things split line by line +PrevIFS=$IFS +IFS=$'\n' +PackageSources+="$IFS" +PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') +IFS=$PrevIFS + +for FeedName in ${PackageSources[@]} ; do + # Check if there is no existing credential for this FeedName + grep -i "<$FeedName>" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding credentials for $FeedName." + + PackageSourceCredentialsNodeFooter="" + NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + + sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile + fi +done + +# Re-enable any entries in disabledPackageSources where the feed name contains darc-int +grep -i "" $ConfigFile +if [ "$?" == "0" ]; then + DisabledDarcIntSources=() + echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" + DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') + for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do + if [[ $DisabledSourceName == darc-int* ]] + then + OldDisableValue="" + NewDisableValue="" + sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile + echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" + fi + done +fi diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 new file mode 100644 index 0000000000000..8943da242f6e9 --- /dev/null +++ b/eng/common/build.ps1 @@ -0,0 +1,161 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string][Alias('c')]$configuration = "Debug", + [string]$platform = $null, + [string] $projects, + [string][Alias('v')]$verbosity = "minimal", + [string] $msbuildEngine = $null, + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, + [switch][Alias('r')]$restore, + [switch] $deployDeps, + [switch][Alias('b')]$build, + [switch] $rebuild, + [switch] $deploy, + [switch][Alias('t')]$test, + [switch] $integrationTest, + [switch] $performanceTest, + [switch] $sign, + [switch] $pack, + [switch] $publish, + [switch] $clean, + [switch][Alias('bl')]$binaryLog, + [switch][Alias('nobl')]$excludeCIBinarylog, + [switch] $ci, + [switch] $prepareMachine, + [string] $runtimeSourceFeed = '', + [string] $runtimeSourceFeedKey = '', + [switch] $excludePrereleaseVS, + [switch] $help, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties +) + +# Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file +# some computer has this env var defined (e.g. Some HP) +if($env:Platform) { + $env:Platform="" +} +function Print-Usage() { + Write-Host "Common settings:" + Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + Write-Host " -binaryLog Output binary log (short: -bl)" + Write-Host " -help Print help and exit" + Write-Host "" + + Write-Host "Actions:" + Write-Host " -restore Restore dependencies (short: -r)" + Write-Host " -build Build solution (short: -b)" + Write-Host " -rebuild Rebuild solution" + Write-Host " -deploy Deploy built VSIXes" + Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" + Write-Host " -test Run all unit tests in the solution (short: -t)" + Write-Host " -integrationTest Run all integration tests in the solution" + Write-Host " -performanceTest Run all performance tests in the solution" + Write-Host " -pack Package build outputs into NuGet packages and Willow components" + Write-Host " -sign Sign build outputs" + Write-Host " -publish Publish artifacts (e.g. symbols)" + Write-Host " -clean Clean the solution" + Write-Host "" + + Write-Host "Advanced settings:" + Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" + Write-Host " -ci Set when running on CI server" + Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" + Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" + Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" + Write-Host "" + + Write-Host "Command line arguments not listed above are passed thru to msbuild." + Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." +} + +. $PSScriptRoot\tools.ps1 + +function InitializeCustomToolset { + if (-not $restore) { + return + } + + $script = Join-Path $EngRoot 'restore-toolset.ps1' + + if (Test-Path $script) { + . $script + } +} + +function Build { + $toolsetBuildProj = InitializeToolset + InitializeCustomToolset + + $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' } + $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' } + + if ($projects) { + # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. + # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty. + [string[]] $msbuildArgs = $properties + + # Resolve relative project paths into full paths + $projects = ($projects.Split(';').ForEach({Resolve-Path $_}) -join ';') + + $msbuildArgs += "/p:Projects=$projects" + $properties = $msbuildArgs + } + + MSBuild $toolsetBuildProj ` + $bl ` + $platformArg ` + /p:Configuration=$configuration ` + /p:RepoRoot=$RepoRoot ` + /p:Restore=$restore ` + /p:DeployDeps=$deployDeps ` + /p:Build=$build ` + /p:Rebuild=$rebuild ` + /p:Deploy=$deploy ` + /p:Test=$test ` + /p:Pack=$pack ` + /p:IntegrationTest=$integrationTest ` + /p:PerformanceTest=$performanceTest ` + /p:Sign=$sign ` + /p:Publish=$publish ` + @properties +} + +try { + if ($clean) { + if (Test-Path $ArtifactsDir) { + Remove-Item -Recurse -Force $ArtifactsDir + Write-Host 'Artifacts directory deleted.' + } + exit 0 + } + + if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { + Print-Usage + exit 0 + } + + if ($ci) { + if (-not $excludeCIBinarylog) { + $binaryLog = $true + } + $nodeReuse = $false + } + + if ($restore) { + InitializeNativeTools + } + + Build +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/eng/common/build.sh b/eng/common/build.sh new file mode 100755 index 0000000000000..55b298f16ccd1 --- /dev/null +++ b/eng/common/build.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash + +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +# Stop script if command returns non-zero exit code. +# Prevents hidden errors caused by missing error code propagation. +set -e + +usage() +{ + echo "Common settings:" + echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)" + echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + echo " --binaryLog Create MSBuild binary log (short: -bl)" + echo " --help Print help and exit (short: -h)" + echo "" + + echo "Actions:" + echo " --restore Restore dependencies (short: -r)" + echo " --build Build solution (short: -b)" + echo " --rebuild Rebuild solution" + echo " --test Run all unit tests in the solution (short: -t)" + echo " --integrationTest Run all integration tests in the solution" + echo " --performanceTest Run all performance tests in the solution" + echo " --pack Package build outputs into NuGet packages and Willow components" + echo " --sign Sign build outputs" + echo " --publish Publish artifacts (e.g. symbols)" + echo " --clean Clean the solution" + echo "" + + echo "Advanced settings:" + echo " --projects Project or solution file(s) to build" + echo " --ci Set when running on CI server" + echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" + echo " --prepareMachine Prepare machine for CI run, clean up processes after build" + echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo "" + echo "Command line arguments not listed above are passed thru to msbuild." + echo "Arguments can also be passed in with a single hyphen." +} + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +restore=false +build=false +rebuild=false +test=false +integration_test=false +performance_test=false +pack=false +publish=false +sign=false +public=false +ci=false +clean=false + +warn_as_error=true +node_reuse=true +binary_log=false +exclude_ci_binary_log=false +pipelines_log=false + +projects='' +configuration='Debug' +prepare_machine=false +verbosity='minimal' +runtime_source_feed='' +runtime_source_feed_key='' + +properties='' +while [[ $# > 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -help|-h) + usage + exit 0 + ;; + -clean) + clean=true + ;; + -configuration|-c) + configuration=$2 + shift + ;; + -verbosity|-v) + verbosity=$2 + shift + ;; + -binarylog|-bl) + binary_log=true + ;; + -excludeCIBinarylog|-nobl) + exclude_ci_binary_log=true + ;; + -pipelineslog|-pl) + pipelines_log=true + ;; + -restore|-r) + restore=true + ;; + -build|-b) + build=true + ;; + -rebuild) + rebuild=true + ;; + -pack) + pack=true + ;; + -test|-t) + test=true + ;; + -integrationtest) + integration_test=true + ;; + -performancetest) + performance_test=true + ;; + -sign) + sign=true + ;; + -publish) + publish=true + ;; + -preparemachine) + prepare_machine=true + ;; + -projects) + projects=$2 + shift + ;; + -ci) + ci=true + ;; + -warnaserror) + warn_as_error=$2 + shift + ;; + -nodereuse) + node_reuse=$2 + shift + ;; + -runtimesourcefeed) + runtime_source_feed=$2 + shift + ;; + -runtimesourcefeedkey) + runtime_source_feed_key=$2 + shift + ;; + *) + properties="$properties $1" + ;; + esac + + shift +done + +if [[ "$ci" == true ]]; then + pipelines_log=true + node_reuse=false + if [[ "$exclude_ci_binary_log" == false ]]; then + binary_log=true + fi +fi + +. "$scriptroot/tools.sh" + +function InitializeCustomToolset { + local script="$eng_root/restore-toolset.sh" + + if [[ -a "$script" ]]; then + . "$script" + fi +} + +function Build { + InitializeToolset + InitializeCustomToolset + + if [[ ! -z "$projects" ]]; then + properties="$properties /p:Projects=$projects" + fi + + local bl="" + if [[ "$binary_log" == true ]]; then + bl="/bl:\"$log_dir/Build.binlog\"" + fi + + MSBuild $_InitializeToolset \ + $bl \ + /p:Configuration=$configuration \ + /p:RepoRoot="$repo_root" \ + /p:Restore=$restore \ + /p:Build=$build \ + /p:Rebuild=$rebuild \ + /p:Test=$test \ + /p:Pack=$pack \ + /p:IntegrationTest=$integration_test \ + /p:PerformanceTest=$performance_test \ + /p:Sign=$sign \ + /p:Publish=$publish \ + $properties + + ExitWithExitCode 0 +} + +if [[ "$clean" == true ]]; then + if [ -d "$artifacts_dir" ]; then + rm -rf $artifacts_dir + echo "Artifacts directory deleted." + fi + exit 0 +fi + +if [[ "$restore" == true ]]; then + InitializeNativeTools +fi + +Build diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh new file mode 100755 index 0000000000000..1a02c0dec8fd7 --- /dev/null +++ b/eng/common/cibuild.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where + # the symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@ \ No newline at end of file diff --git a/eng/common/cross/arm/sources.list.bionic b/eng/common/cross/arm/sources.list.bionic new file mode 100644 index 0000000000000..2109557409576 --- /dev/null +++ b/eng/common/cross/arm/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.jessie b/eng/common/cross/arm/sources.list.jessie new file mode 100644 index 0000000000000..4d142ac9b1089 --- /dev/null +++ b/eng/common/cross/arm/sources.list.jessie @@ -0,0 +1,3 @@ +# Debian (sid) # UNSTABLE +deb http://ftp.debian.org/debian/ sid main contrib non-free +deb-src http://ftp.debian.org/debian/ sid main contrib non-free diff --git a/eng/common/cross/arm/sources.list.xenial b/eng/common/cross/arm/sources.list.xenial new file mode 100644 index 0000000000000..eacd86b7df3c1 --- /dev/null +++ b/eng/common/cross/arm/sources.list.xenial @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm/sources.list.zesty b/eng/common/cross/arm/sources.list.zesty new file mode 100644 index 0000000000000..ea2c14a787473 --- /dev/null +++ b/eng/common/cross/arm/sources.list.zesty @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/arm/tizen-build-rootfs.sh b/eng/common/cross/arm/tizen-build-rootfs.sh new file mode 100644 index 0000000000000..9fdb32e920e22 --- /dev/null +++ b/eng/common/cross/arm/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__ARM_HARDFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__ARM_HARDFP_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__ARM_HARDFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7hl lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm/tizen/tizen.patch b/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 0000000000000..fb12ade7250ae --- /dev/null +++ b/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/eng/common/cross/arm64/sources.list.bionic b/eng/common/cross/arm64/sources.list.bionic new file mode 100644 index 0000000000000..2109557409576 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.buster b/eng/common/cross/arm64/sources.list.buster new file mode 100644 index 0000000000000..7194ac64a9602 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.buster @@ -0,0 +1,11 @@ +deb http://deb.debian.org/debian buster main +deb-src http://deb.debian.org/debian buster main + +deb http://deb.debian.org/debian-security/ buster/updates main +deb-src http://deb.debian.org/debian-security/ buster/updates main + +deb http://deb.debian.org/debian buster-updates main +deb-src http://deb.debian.org/debian buster-updates main + +deb http://deb.debian.org/debian buster-backports main contrib non-free +deb-src http://deb.debian.org/debian buster-backports main contrib non-free diff --git a/eng/common/cross/arm64/sources.list.stretch b/eng/common/cross/arm64/sources.list.stretch new file mode 100644 index 0000000000000..0e1215774368a --- /dev/null +++ b/eng/common/cross/arm64/sources.list.stretch @@ -0,0 +1,12 @@ +deb http://deb.debian.org/debian stretch main +deb-src http://deb.debian.org/debian stretch main + +deb http://deb.debian.org/debian-security/ stretch/updates main +deb-src http://deb.debian.org/debian-security/ stretch/updates main + +deb http://deb.debian.org/debian stretch-updates main +deb-src http://deb.debian.org/debian stretch-updates main + +deb http://deb.debian.org/debian stretch-backports main contrib non-free +deb-src http://deb.debian.org/debian stretch-backports main contrib non-free + diff --git a/eng/common/cross/arm64/sources.list.xenial b/eng/common/cross/arm64/sources.list.xenial new file mode 100644 index 0000000000000..eacd86b7df3c1 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.xenial @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm64/sources.list.zesty b/eng/common/cross/arm64/sources.list.zesty new file mode 100644 index 0000000000000..ea2c14a787473 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.zesty @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/tizen-build-rootfs.sh b/eng/common/cross/arm64/tizen-build-rootfs.sh new file mode 100644 index 0000000000000..13bfddb5e2a70 --- /dev/null +++ b/eng/common/cross/arm64/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm64 ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen/ +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch new file mode 100644 index 0000000000000..af7c8be059068 --- /dev/null +++ b/eng/common/cross/arm64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-littleaarch64) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) ) diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch new file mode 100644 index 0000000000000..2d2615619351f --- /dev/null +++ b/eng/common/cross/armel/armel.jessie.patch @@ -0,0 +1,43 @@ +diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h +--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700 ++++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700 +@@ -69,10 +69,10 @@ + #endif + #ifdef UATOMIC_HAS_ATOMIC_SHORT + case 2: +- return __sync_val_compare_and_swap_2(addr, old, _new); ++ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new); + #endif + case 4: +- return __sync_val_compare_and_swap_4(addr, old, _new); ++ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new); + #if (CAA_BITS_PER_LONG == 64) + case 8: + return __sync_val_compare_and_swap_8(addr, old, _new); +@@ -109,7 +109,7 @@ + return; + #endif + case 4: +- __sync_and_and_fetch_4(addr, val); ++ __sync_and_and_fetch_4((uint32_t*) addr, val); + return; + #if (CAA_BITS_PER_LONG == 64) + case 8: +@@ -148,7 +148,7 @@ + return; + #endif + case 4: +- __sync_or_and_fetch_4(addr, val); ++ __sync_or_and_fetch_4((uint32_t*) addr, val); + return; + #if (CAA_BITS_PER_LONG == 64) + case 8: +@@ -187,7 +187,7 @@ + return __sync_add_and_fetch_2(addr, val); + #endif + case 4: +- return __sync_add_and_fetch_4(addr, val); ++ return __sync_add_and_fetch_4((uint32_t*) addr, val); + #if (CAA_BITS_PER_LONG == 64) + case 8: + return __sync_add_and_fetch_8(addr, val); diff --git a/eng/common/cross/armel/sources.list.jessie b/eng/common/cross/armel/sources.list.jessie new file mode 100644 index 0000000000000..3d9c3059d8970 --- /dev/null +++ b/eng/common/cross/armel/sources.list.jessie @@ -0,0 +1,3 @@ +# Debian (jessie) # Stable +deb http://ftp.debian.org/debian/ jessie main contrib non-free +deb-src http://ftp.debian.org/debian/ jessie main contrib non-free diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh new file mode 100755 index 0000000000000..9a4438af61c21 --- /dev/null +++ b/eng/common/cross/armel/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__ARM_SOFTFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__ARM_SOFTFP_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__ARM_SOFTFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/armel/tizen/tizen-dotnet.ks b/eng/common/cross/armel/tizen/tizen-dotnet.ks new file mode 100644 index 0000000000000..506d455bd4ff8 --- /dev/null +++ b/eng/common/cross/armel/tizen/tizen-dotnet.ks @@ -0,0 +1,50 @@ +lang en_US.UTF-8 +keyboard us +timezone --utc Asia/Seoul + +part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime + +rootpw tizen +desktop --autologinuser=root +user --name root --groups audio,video --password 'tizen' + +repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no +repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no + +%packages +tar +gzip + +sed +grep +gawk +perl + +binutils +findutils +util-linux +lttng-ust +userspace-rcu +procps-ng +tzdata +ca-certificates + + +### Core FX +libicu +libunwind +iputils +zlib +krb5 +libcurl +libopenssl + +%end + +%post + +### Update /tmp privilege +chmod 777 /tmp +#################################### + +%end diff --git a/eng/common/cross/armel/tizen/tizen.patch b/eng/common/cross/armel/tizen/tizen.patch new file mode 100644 index 0000000000000..ca7c7c1ff751f --- /dev/null +++ b/eng/common/cross/armel/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster new file mode 100644 index 0000000000000..f27fc4fb346b6 --- /dev/null +++ b/eng/common/cross/armv6/sources.list.buster @@ -0,0 +1,2 @@ +deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi +deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh new file mode 100755 index 0000000000000..42516bbeebc3f --- /dev/null +++ b/eng/common/cross/build-android-rootfs.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -e +__NDK_Version=r21 + +usage() +{ + echo "Creates a toolchain and sysroot used for cross-compiling for Android." + echo. + echo "Usage: $0 [BuildArch] [ApiLevel]" + echo. + echo "BuildArch is the target architecture of Android. Currently only arm64 is supported." + echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html" + echo. + echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior" + echo "by setting the TOOLCHAIN_DIR environment variable" + echo. + echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," + echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." + echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android." + exit 1 +} + +__ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h +__BuildArch=arm64 +__AndroidArch=aarch64 +__AndroidToolchain=aarch64-linux-android + +for i in "$@" + do + lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" + case $lowerI in + -?|-h|--help) + usage + exit 1 + ;; + arm64) + __BuildArch=arm64 + __AndroidArch=aarch64 + __AndroidToolchain=aarch64-linux-android + ;; + arm) + __BuildArch=arm + __AndroidArch=arm + __AndroidToolchain=arm-linux-androideabi + ;; + *[0-9]) + __ApiLevel=$i + ;; + *) + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + ;; + esac +done + +# Obtain the location of the bash script to figure out where the root of the repo is. +__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +__CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs" + +if [[ ! -f "$__CrossDir" ]]; then + mkdir -p "$__CrossDir" +fi + +# Resolve absolute path to avoid `../` in build logs +__CrossDir="$( cd "$__CrossDir" && pwd )" + +__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version" +__lldb_Dir="$__CrossDir/lldb" +__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version" + +if [[ -n "$TOOLCHAIN_DIR" ]]; then + __ToolchainDir=$TOOLCHAIN_DIR +fi + +if [[ -n "$NDK_DIR" ]]; then + __NDK_Dir=$NDK_DIR +fi + +echo "Target API level: $__ApiLevel" +echo "Target architecture: $__BuildArch" +echo "NDK location: $__NDK_Dir" +echo "Target Toolchain location: $__ToolchainDir" + +# Download the NDK if required +if [ ! -d $__NDK_Dir ]; then + echo Downloading the NDK into $__NDK_Dir + mkdir -p $__NDK_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip + unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir +fi + +if [ ! -d $__lldb_Dir ]; then + mkdir -p $__lldb_Dir + echo Downloading LLDB into $__lldb_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip + unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir +fi + +echo "Download dependencies..." +__TmpDir=$__CrossDir/tmp/$__BuildArch/ +mkdir -p "$__TmpDir" + +# combined dependencies for coreclr, installer and libraries +__AndroidPackages="libicu" +__AndroidPackages+=" libandroid-glob" +__AndroidPackages+=" liblzma" +__AndroidPackages+=" krb5" +__AndroidPackages+=" openssl" + +for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ + grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do + + if [[ "$path" != "Filename:" ]]; then + echo "Working on: $path" + wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" + fi +done + +cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" + +# Generate platform file for build.sh script to assign to __DistroRid +echo "Generating platform file..." +echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform + +echo "Now to build coreclr, libraries and installers; run:" +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory coreclr +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory libraries +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory installer diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh new file mode 100755 index 0000000000000..e784c9c005a2a --- /dev/null +++ b/eng/common/cross/build-rootfs.sh @@ -0,0 +1,383 @@ +#!/usr/bin/env bash + +set -e + +usage() +{ + echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" + echo "BuildArch can be: arm(default), armel, arm64, x86" + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd12, freebsd13" + echo " for illumos can be: illumos." + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" + echo "--skipunmount - optional, will skip the unmount of rootfs folder." + echo "--use-mirror - optional, use mirror URL to fetch resources, when available." + exit 1 +} + +__CodeName=xenial +__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__InitialDir=$PWD +__BuildArch=arm +__AlpineArch=armv7 +__QEMUArch=arm +__UbuntuArch=armhf +__UbuntuRepo="http://ports.ubuntu.com/" +__LLDB_Package="liblldb-3.9-dev" +__SkipUnmount=0 + +# base development support +__UbuntuPackages="build-essential" + +__AlpinePackages="alpine-base" +__AlpinePackages+=" build-base" +__AlpinePackages+=" linux-headers" +__AlpinePackages+=" lldb-dev" +__AlpinePackages+=" python3" +__AlpinePackages+=" libedit" + +# symlinks fixer +__UbuntuPackages+=" symlinks" + +# CoreCLR and CoreFX dependencies +__UbuntuPackages+=" libicu-dev" +__UbuntuPackages+=" liblttng-ust-dev" +__UbuntuPackages+=" libunwind8-dev" + +__AlpinePackages+=" gettext-dev" +__AlpinePackages+=" icu-dev" +__AlpinePackages+=" libunwind-dev" +__AlpinePackages+=" lttng-ust-dev" + +# CoreFX dependencies +__UbuntuPackages+=" libcurl4-openssl-dev" +__UbuntuPackages+=" libkrb5-dev" +__UbuntuPackages+=" libssl-dev" +__UbuntuPackages+=" zlib1g-dev" + +__AlpinePackages+=" curl-dev" +__AlpinePackages+=" krb5-dev" +__AlpinePackages+=" openssl-dev" +__AlpinePackages+=" zlib-dev" + +__FreeBSDBase="12.3-RELEASE" +__FreeBSDPkg="1.17.0" +__FreeBSDABI="12" +__FreeBSDPackages="libunwind" +__FreeBSDPackages+=" icu" +__FreeBSDPackages+=" libinotify" +__FreeBSDPackages+=" openssl" +__FreeBSDPackages+=" krb5" +__FreeBSDPackages+=" terminfo-db" + +__IllumosPackages="icu-64.2nb2" +__IllumosPackages+=" mit-krb5-1.16.2nb4" +__IllumosPackages+=" openssl-1.1.1e" +__IllumosPackages+=" zlib-1.2.11" + +# ML.NET dependencies +__UbuntuPackages+=" libomp5" +__UbuntuPackages+=" libomp-dev" + +__UseMirror=0 + +__UnprocessedBuildArgs= +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + -?|-h|--help) + usage + exit 1 + ;; + arm) + __BuildArch=arm + __UbuntuArch=armhf + __AlpineArch=armv7 + __QEMUArch=arm + ;; + armv6) + __BuildArch=armv6 + __UbuntuArch=armhf + __QEMUArch=arm + __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" + __CodeName=buster + __LLDB_Package="liblldb-6.0-dev" + __Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg" + ;; + arm64) + __BuildArch=arm64 + __UbuntuArch=arm64 + __AlpineArch=aarch64 + __QEMUArch=aarch64 + ;; + armel) + __BuildArch=armel + __UbuntuArch=armel + __UbuntuRepo="http://ftp.debian.org/debian/" + __CodeName=jessie + ;; + ppc64le) + __BuildArch=ppc64le + __UbuntuArch=ppc64el + __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') + unset __LLDB_Package + ;; + s390x) + __BuildArch=s390x + __UbuntuArch=s390x + __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') + unset __LLDB_Package + ;; + x86) + __BuildArch=x86 + __UbuntuArch=i386 + __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + ;; + lldb3.6) + __LLDB_Package="lldb-3.6-dev" + ;; + lldb3.8) + __LLDB_Package="lldb-3.8-dev" + ;; + lldb3.9) + __LLDB_Package="liblldb-3.9-dev" + ;; + lldb4.0) + __LLDB_Package="liblldb-4.0-dev" + ;; + lldb5.0) + __LLDB_Package="liblldb-5.0-dev" + ;; + lldb6.0) + __LLDB_Package="liblldb-6.0-dev" + ;; + no-lldb) + unset __LLDB_Package + ;; + xenial) # Ubuntu 16.04 + if [ "$__CodeName" != "jessie" ]; then + __CodeName=xenial + fi + ;; + zesty) # Ubuntu 17.04 + if [ "$__CodeName" != "jessie" ]; then + __CodeName=zesty + fi + ;; + bionic) # Ubuntu 18.04 + if [ "$__CodeName" != "jessie" ]; then + __CodeName=bionic + fi + ;; + jessie) # Debian 8 + __CodeName=jessie + __UbuntuRepo="http://ftp.debian.org/debian/" + ;; + stretch) # Debian 9 + __CodeName=stretch + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; + buster) # Debian 10 + __CodeName=buster + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; + tizen) + if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ] && [ "$__BuildArch" != "x86" ] ; then + echo "Tizen is available only for arm, armel, arm64 and x86." + usage; + exit 1; + fi + __CodeName= + __UbuntuRepo= + __Tizen=tizen + ;; + alpine|alpine3.13) + __CodeName=alpine + __UbuntuRepo= + __AlpineVersion=3.13 + __AlpinePackages+=" llvm10-libs" + ;; + alpine3.14) + __CodeName=alpine + __UbuntuRepo= + __AlpineVersion=3.14 + __AlpinePackages+=" llvm11-libs" + ;; + freebsd12) + __CodeName=freebsd + __BuildArch=x64 + __SkipUnmount=1 + ;; + freebsd13) + __CodeName=freebsd + __FreeBSDBase="13.0-RELEASE" + __FreeBSDABI="13" + __BuildArch=x64 + __SkipUnmount=1 + ;; + illumos) + __CodeName=illumos + __BuildArch=x64 + __SkipUnmount=1 + ;; + --skipunmount) + __SkipUnmount=1 + ;; + --rootfsdir|-rootfsdir) + shift + __RootfsDir=$1 + ;; + --use-mirror) + __UseMirror=1 + ;; + *) + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" + ;; + esac + + shift +done + +if [ -e "$__Keyring" ]; then + __Keyring="--keyring=$__Keyring" +else + __Keyring="" +fi + +if [ "$__BuildArch" == "armel" ]; then + __LLDB_Package="lldb-3.5-dev" +fi +__UbuntuPackages+=" ${__LLDB_Package:-}" + +if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then + __RootfsDir=$ROOTFS_DIR +fi + +if [ -z "$__RootfsDir" ]; then + __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" +fi + +if [ -d "$__RootfsDir" ]; then + if [ $__SkipUnmount == 0 ]; then + umount $__RootfsDir/* || true + fi + rm -rf $__RootfsDir +fi + +mkdir -p $__RootfsDir +__RootfsDir="$( cd "$__RootfsDir" && pwd )" + +if [[ "$__CodeName" == "alpine" ]]; then + __ApkToolsVersion=2.9.1 + __ApkToolsDir=$(mktemp -d) + wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir + tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir + mkdir -p $__RootfsDir/usr/bin + cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin + + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ + -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackages + + rm -r $__ApkToolsDir +elif [[ "$__CodeName" == "freebsd" ]]; then + mkdir -p $__RootfsDir/usr/local/etc + JOBS="$(getconf _NPROCESSORS_ONLN)" + wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf + echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf + mkdir -p $__RootfsDir/tmp + # get and build package manager + wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - + cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + # needed for install to succeed + mkdir -p $__RootfsDir/host/etc + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install + rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + # install packages we need. + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "illumos" ]]; then + mkdir "$__RootfsDir/tmp" + pushd "$__RootfsDir/tmp" + JOBS="$(getconf _NPROCESSORS_ONLN)" + echo "Downloading sysroot." + wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + echo "Building binutils. Please wait.." + wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + mkdir build-binutils && cd build-binutils + ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" + make -j "$JOBS" && make install && cd .. + echo "Building gcc. Please wait.." + wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + CFLAGS="-fPIC" + CXXFLAGS="-fPIC" + CXXFLAGS_FOR_TARGET="-fPIC" + CFLAGS_FOR_TARGET="-fPIC" + export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET + mkdir build-gcc && cd build-gcc + ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ + --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ + --disable-libquadmath-support --disable-shared --enable-tls + make -j "$JOBS" && make install && cd .. + BaseUrl=https://pkgsrc.joyent.com + if [[ "$__UseMirror" == 1 ]]; then + BaseUrl=http://pkgsrc.smartos.skylime.net + fi + BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All + echo "Downloading dependencies." + read -ra array <<<"$__IllumosPackages" + for package in "${array[@]}"; do + echo "Installing $package..." + wget "$BaseUrl"/"$package".tgz + ar -x "$package".tgz + tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null + done + echo "Cleaning up temporary files." + popd + rm -rf "$__RootfsDir"/{tmp,+*} + mkdir -p "$__RootfsDir"/usr/include/net + mkdir -p "$__RootfsDir"/usr/include/netpacket + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h +elif [[ -n $__CodeName ]]; then + qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list + chroot $__RootfsDir apt-get update + chroot $__RootfsDir apt-get -f -y install + chroot $__RootfsDir apt-get -y install $__UbuntuPackages + chroot $__RootfsDir symlinks -cr /usr + chroot $__RootfsDir apt-get clean + + if [ $__SkipUnmount == 0 ]; then + umount $__RootfsDir/* || true + fi + + if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then + pushd $__RootfsDir + patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch + popd + fi +elif [[ "$__Tizen" == "tizen" ]]; then + ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh +else + echo "Unsupported target platform." + usage; + exit 1 +fi diff --git a/eng/common/cross/ppc64le/sources.list.bionic b/eng/common/cross/ppc64le/sources.list.bionic new file mode 100644 index 0000000000000..2109557409576 --- /dev/null +++ b/eng/common/cross/ppc64le/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/s390x/sources.list.bionic b/eng/common/cross/s390x/sources.list.bionic new file mode 100644 index 0000000000000..2109557409576 --- /dev/null +++ b/eng/common/cross/s390x/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake new file mode 100644 index 0000000000000..eaeeab38fa1f3 --- /dev/null +++ b/eng/common/cross/toolchain.cmake @@ -0,0 +1,286 @@ +set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) + +set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) +if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) + set(CMAKE_SYSTEM_NAME FreeBSD) + set(FREEBSD 1) +elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) + set(CMAKE_SYSTEM_NAME SunOS) + set(ILLUMOS 1) +else() + set(CMAKE_SYSTEM_NAME Linux) + set(LINUX 1) +endif() +set(CMAKE_SYSTEM_VERSION 1) + +if(EXISTS ${CROSS_ROOTFS}/etc/tizen-release) + set(TIZEN 1) +elseif(EXISTS ${CROSS_ROOTFS}/android_platform) + set(ANDROID 1) +endif() + +if(TARGET_ARCH_NAME STREQUAL "armel") + set(CMAKE_SYSTEM_PROCESSOR armv7l) + set(TOOLCHAIN "arm-linux-gnueabi") + if(TIZEN) + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm") + set(CMAKE_SYSTEM_PROCESSOR armv7l) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf) + set(TOOLCHAIN "armv7-alpine-linux-musleabihf") + elseif(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) + set(TOOLCHAIN "armv6-alpine-linux-musleabihf") + else() + set(TOOLCHAIN "arm-linux-gnueabihf") + endif() + if(TIZEN) + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "armv6") + set(CMAKE_SYSTEM_PROCESSOR armv6l) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) + set(TOOLCHAIN "armv6-alpine-linux-musleabihf") + else() + set(TOOLCHAIN "arm-linux-gnueabihf") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm64") + set(CMAKE_SYSTEM_PROCESSOR aarch64) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) + set(TOOLCHAIN "aarch64-alpine-linux-musl") + else() + set(TOOLCHAIN "aarch64-linux-gnu") + endif() + if(TIZEN) + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") + set(CMAKE_SYSTEM_PROCESSOR ppc64le) + set(TOOLCHAIN "powerpc64le-linux-gnu") +elseif(TARGET_ARCH_NAME STREQUAL "s390x") + set(CMAKE_SYSTEM_PROCESSOR s390x) + set(TOOLCHAIN "s390x-linux-gnu") +elseif(TARGET_ARCH_NAME STREQUAL "x86") + set(CMAKE_SYSTEM_PROCESSOR i686) + set(TOOLCHAIN "i686-linux-gnu") + if(TIZEN) + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") + endif() +elseif (FREEBSD) + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(triple "x86_64-unknown-freebsd12") +elseif (ILLUMOS) + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(TOOLCHAIN "x86_64-illumos") +else() + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, armv6, arm64, ppc64le, s390x and x86 are supported!") +endif() + +if(DEFINED ENV{TOOLCHAIN}) + set(TOOLCHAIN $ENV{TOOLCHAIN}) +endif() + +# Specify include paths +if(TIZEN) + if(TARGET_ARCH_NAME STREQUAL "arm") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) + endif() + if(TARGET_ARCH_NAME STREQUAL "armel") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) + endif() + if(TARGET_ARCH_NAME STREQUAL "arm64") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) + endif() + if(TARGET_ARCH_NAME STREQUAL "x86") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) + endif() +endif() + +if(ANDROID) + if(TARGET_ARCH_NAME STREQUAL "arm") + set(ANDROID_ABI armeabi-v7a) + elseif(TARGET_ARCH_NAME STREQUAL "arm64") + set(ANDROID_ABI arm64-v8a) + endif() + + # extract platform number required by the NDK's toolchain + file(READ "${CROSS_ROOTFS}/android_platform" RID_FILE_CONTENTS) + string(REPLACE "RID=" "" ANDROID_RID "${RID_FILE_CONTENTS}") + string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "${ANDROID_RID}") + + set(ANDROID_TOOLCHAIN clang) + set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository + set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib") + set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include") + + # include official NDK toolchain script + include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) +elseif(FREEBSD) + # we cross-compile by instructing clang + set(CMAKE_C_COMPILER_TARGET ${triple}) + set(CMAKE_CXX_COMPILER_TARGET ${triple}) + set(CMAKE_ASM_COMPILER_TARGET ${triple}) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") +elseif(ILLUMOS) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + include_directories(SYSTEM ${CROSS_ROOTFS}/include) + + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + function(locate_toolchain_exec exec var) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + find_program(EXEC_LOCATION_${exec} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) + endfunction() + + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") +else() + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") +endif() + +# Specify link flags + +function(add_toolchain_linker_flag Flag) + set(Config "${ARGV1}") + set(CONFIG_SUFFIX "") + if (NOT Config STREQUAL "") + set(CONFIG_SUFFIX "_${Config}") + endif() + set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) + set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) +endfunction() + +if(LINUX) + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}") +endif() + +if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") + if(TIZEN) + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm64") + if(TIZEN) + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "x86") + add_toolchain_linker_flag(-m32) + + if(TIZEN) + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + endif() +elseif(ILLUMOS) + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") +endif() + +# Specify compile options + +if((TARGET_ARCH_NAME MATCHES "^(arm|armv6|armel|arm64|ppc64le|s390x)$" AND NOT ANDROID) OR ILLUMOS) + set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) +endif() + +if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") + add_compile_options(-mthumb) + if (NOT DEFINED CLR_ARM_FPU_TYPE) + set (CLR_ARM_FPU_TYPE vfpv3) + endif (NOT DEFINED CLR_ARM_FPU_TYPE) + + add_compile_options (-mfpu=${CLR_ARM_FPU_TYPE}) + if (NOT DEFINED CLR_ARM_FPU_CAPABILITY) + set (CLR_ARM_FPU_CAPABILITY 0x7) + endif (NOT DEFINED CLR_ARM_FPU_CAPABILITY) + + add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY}) + + if(TARGET_ARCH_NAME STREQUAL "armel") + add_compile_options(-mfloat-abi=softfp) + endif() +elseif(TARGET_ARCH_NAME STREQUAL "x86") + add_compile_options(-m32) + add_compile_options(-Wno-error=unused-command-line-argument) +endif() + +if(TIZEN) + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|x86)$") + add_compile_options(-Wno-deprecated-declarations) # compile-time option + add_compile_options(-D__extern_always_inline=inline) # compile-time option + endif() +endif() + +# Set LLDB include and library paths for builds that need lldb. +if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") + if(TARGET_ARCH_NAME STREQUAL "x86") + set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}") + else() # arm/armel case + set(LLVM_CROSS_DIR "$ENV{LLVM_ARM_HOME}") + endif() + if(LLVM_CROSS_DIR) + set(WITH_LLDB_LIBS "${LLVM_CROSS_DIR}/lib/" CACHE STRING "") + set(WITH_LLDB_INCLUDES "${LLVM_CROSS_DIR}/include" CACHE STRING "") + set(LLDB_H "${WITH_LLDB_INCLUDES}" CACHE STRING "") + set(LLDB "${LLVM_CROSS_DIR}/lib/liblldb.so" CACHE STRING "") + else() + if(TARGET_ARCH_NAME STREQUAL "x86") + set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/i386-linux-gnu" CACHE STRING "") + set(CHECK_LLVM_DIR "${CROSS_ROOTFS}/usr/lib/llvm-3.8/include") + if(EXISTS "${CHECK_LLVM_DIR}" AND IS_DIRECTORY "${CHECK_LLVM_DIR}") + set(WITH_LLDB_INCLUDES "${CHECK_LLVM_DIR}") + else() + set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include") + endif() + else() # arm/armel case + set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}" CACHE STRING "") + set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include" CACHE STRING "") + endif() + endif() +endif() + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/x86/sources.list.bionic b/eng/common/cross/x86/sources.list.bionic new file mode 100644 index 0000000000000..a71ccadcffaf8 --- /dev/null +++ b/eng/common/cross/x86/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.xenial b/eng/common/cross/x86/sources.list.xenial new file mode 100644 index 0000000000000..ad9c5a0144ef0 --- /dev/null +++ b/eng/common/cross/x86/sources.list.xenial @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/x86/tizen-build-rootfs.sh b/eng/common/cross/x86/tizen-build-rootfs.sh new file mode 100644 index 0000000000000..f5f955dc8f267 --- /dev/null +++ b/eng/common/cross/x86/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__X86_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__X86_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__X86_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-x86 ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize i686 base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs i686 gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs i686 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs i686 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs i686 gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/x86/tizen/tizen.patch b/eng/common/cross/x86/tizen/tizen.patch new file mode 100644 index 0000000000000..f4fe8838ad668 --- /dev/null +++ b/eng/common/cross/x86/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-i386) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.2 ) ) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 new file mode 100644 index 0000000000000..435e7641341b1 --- /dev/null +++ b/eng/common/darc-init.ps1 @@ -0,0 +1,47 @@ +param ( + $darcVersion = $null, + $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16', + $verbosity = 'minimal', + $toolpath = $null +) + +. $PSScriptRoot\tools.ps1 + +function InstallDarcCli ($darcVersion, $toolpath) { + $darcCliPackageName = 'microsoft.dotnet.darc' + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list -g + + if ($toolList -like "*$darcCliPackageName*") { + & "$dotnet" tool uninstall $darcCliPackageName -g + } + + # If the user didn't explicitly specify the darc version, + # query the Maestro API for the correct version of darc to install. + if (-not $darcVersion) { + $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content + } + + $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + + Write-Host "Installing Darc CLI version $darcVersion..." + Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' + if (-not $toolpath) { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + }else { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + } +} + +try { + InstallDarcCli $darcVersion $toolpath +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Darc' -Message $_ + ExitWithExitCode 1 +} \ No newline at end of file diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh new file mode 100755 index 0000000000000..84c1d0cc2e75a --- /dev/null +++ b/eng/common/darc-init.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +darcVersion='' +versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16' +verbosity='minimal' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + --darcversion) + darcVersion=$2 + shift + ;; + --versionendpoint) + versionEndpoint=$2 + shift + ;; + --verbosity) + verbosity=$2 + shift + ;; + --toolpath) + toolpath=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ -z "$darcVersion" ]; then + darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") +fi + +function InstallDarcCli { + local darc_cli_package_name="microsoft.dotnet.darc" + + InitializeDotNetCli true + local dotnet_root=$_InitializeDotNetCli + + if [ -z "$toolpath" ]; then + local tool_list=$($dotnet_root/dotnet tool list -g) + if [[ $tool_list = *$darc_cli_package_name* ]]; then + echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) + fi + else + local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath") + if [[ $tool_list = *$darc_cli_package_name* ]]; then + echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath") + fi + fi + + local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" + + echo "Installing Darc CLI version $darcVersion..." + echo "You may need to restart your command shell if this is the first dotnet tool you have installed." + if [ -z "$toolpath" ]; then + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + else + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + fi +} + +InstallDarcCli diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd new file mode 100644 index 0000000000000..b1c2642e76f72 --- /dev/null +++ b/eng/common/dotnet-install.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 new file mode 100644 index 0000000000000..811f0f717f736 --- /dev/null +++ b/eng/common/dotnet-install.ps1 @@ -0,0 +1,28 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = 'minimal', + [string] $architecture = '', + [string] $version = 'Latest', + [string] $runtime = 'dotnet', + [string] $RuntimeSourceFeed = '', + [string] $RuntimeSourceFeedKey = '' +) + +. $PSScriptRoot\tools.ps1 + +$dotnetRoot = Join-Path $RepoRoot '.dotnet' + +$installdir = $dotnetRoot +try { + if ($architecture -and $architecture.Trim() -eq 'x86') { + $installdir = Join-Path $installdir 'x86' + } + InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh new file mode 100755 index 0000000000000..5c94e98632a0a --- /dev/null +++ b/eng/common/dotnet-install.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +version='Latest' +architecture='' +runtime='dotnet' +runtimeSourceFeed='' +runtimeSourceFeedKey='' +while [[ $# > 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -version|-v) + shift + version="$1" + ;; + -architecture|-a) + shift + architecture="$1" + ;; + -runtime|-r) + shift + runtime="$1" + ;; + -runtimesourcefeed) + shift + runtimeSourceFeed="$1" + ;; + -runtimesourcefeedkey) + shift + runtimeSourceFeedKey="$1" + ;; + *) + Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1" + exit 1 + ;; + esac + shift +done + +# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples +cpuname=$(uname -m) +case $cpuname in + aarch64) + buildarch=arm64 + ;; + loongarch64) + buildarch=loongarch64 + ;; + amd64|x86_64) + buildarch=x64 + ;; + armv*l) + buildarch=arm + ;; + i686) + buildarch=x86 + ;; + *) + echo "Unknown CPU $cpuname detected, treating it as x64" + buildarch=x64 + ;; +esac + +dotnetRoot="${repo_root}.dotnet" +if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then + dotnetRoot="$dotnetRoot/$architecture" +fi + +InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || { + local exit_code=$? + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} + +ExitWithExitCode 0 diff --git a/eng/common/enable-cross-org-publishing.ps1 b/eng/common/enable-cross-org-publishing.ps1 new file mode 100644 index 0000000000000..da09da4f1fc44 --- /dev/null +++ b/eng/common/enable-cross-org-publishing.ps1 @@ -0,0 +1,13 @@ +param( + [string] $token +) + + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +# Write-PipelineSetVariable will no-op if a variable named $ci is not defined +# Since this script is only ever called in AzDO builds, just universally set it +$ci = $true + +Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 new file mode 100644 index 0000000000000..25e97ac00772d --- /dev/null +++ b/eng/common/generate-locproject.ps1 @@ -0,0 +1,117 @@ +Param( + [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here + [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json + [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one + [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally +) + +# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: +# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" +$exclusions = @{ Exclusions = @() } +if (Test-Path -Path $exclusionsFilePath) +{ + $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json +} + +Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work + +# Template files +$jsonFiles = @() +$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern +$jsonTemplateFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" + $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern + +$xlfFiles = @() + +$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" +$langXlfFiles = @() +if ($allXlfFiles) { + $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' + $firstLangCode = $Matches.1 + $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" +} +$langXlfFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" + $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles + +$locJson = @{ + Projects = @( + @{ + LanguageSet = $LanguageSet + LocItems = @( + $locFiles | ForEach-Object { + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($outputPath.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { + Remove-Item -Path $sourceFile + } + if ($continue) + { + if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" + } + } + else { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } + } + } + } + ) + } + ) +} + +$json = ConvertTo-Json $locJson -Depth 5 +Write-Host "LocProject.json generated:`n`n$json`n`n" +Pop-Location + +if (!$UseCheckedInLocProjectJson) { + New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json +} +else { + New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json + + if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { + Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." + + exit 1 + } + else { + Write-Host "Generated LocProject.json and current LocProject.json are identical." + } +} \ No newline at end of file diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 new file mode 100644 index 0000000000000..a733a8885824a --- /dev/null +++ b/eng/common/generate-sbom-prep.ps1 @@ -0,0 +1,19 @@ +Param( + [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed +) + +Write-Host "Creating dir $ManifestDirPath" +# create directory for sbom manifest to be placed +if (!(Test-Path -path $ManifestDirPath)) +{ + New-Item -ItemType Directory -path $ManifestDirPath + Write-Host "Successfully created directory $ManifestDirPath" +} +else{ + Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." +} + +Write-Host "Updating artifact name" +$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_' +Write-Host "Artifact name $artifact_name" +Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh new file mode 100644 index 0000000000000..f6c77453142a2 --- /dev/null +++ b/eng/common/generate-sbom-prep.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +manifest_dir=$1 + +if [ ! -d "$manifest_dir" ] ; then + mkdir -p "$manifest_dir" + echo "Sbom directory created." $manifest_dir +else + Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." +fi + +artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" +echo "Artifact name before : "$artifact_name +# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. +safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" +echo "Artifact name after : "$safe_artifact_name +export ARTIFACT_NAME=$safe_artifact_name +echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" + +exit 0 diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj new file mode 100644 index 0000000000000..d7f185856e791 --- /dev/null +++ b/eng/common/helixpublish.proj @@ -0,0 +1,26 @@ + + + + msbuild + + + + + %(Identity) + + + + + + $(WorkItemDirectory) + $(WorkItemCommand) + $(WorkItemTimeout) + + + + + + + + + diff --git a/eng/common/init-tools-native.cmd b/eng/common/init-tools-native.cmd new file mode 100644 index 0000000000000..438cd548c452c --- /dev/null +++ b/eng/common/init-tools-native.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" +exit /b %ErrorLevel% \ No newline at end of file diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 new file mode 100644 index 0000000000000..db830c00a6f8d --- /dev/null +++ b/eng/common/init-tools-native.ps1 @@ -0,0 +1,152 @@ +<# +.SYNOPSIS +Entry point script for installing native tools + +.DESCRIPTION +Reads $RepoRoot\global.json file to determine native assets to install +and executes installers for those tools + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER InstallDirectory +Directory to install native toolset. This is a command-line override for the default +Install directory precedence order: +- InstallDirectory command-line override +- NETCOREENG_INSTALL_DIRECTORY environment variable +- (default) %USERPROFILE%/.netcoreeng/native + +.PARAMETER Clean +Switch specifying to not install anything, but cleanup native asset folders + +.PARAMETER Force +Clean and then install tools + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.PARAMETER GlobalJsonFile +File path to global.json file + +.NOTES +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external', + [string] $InstallDirectory, + [switch] $Clean = $False, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [string] $GlobalJsonFile +) + +if (!$GlobalJsonFile) { + $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json' +} + +Set-StrictMode -version 2.0 +$ErrorActionPreference='Stop' + +. $PSScriptRoot\pipeline-logging-functions.ps1 +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq 'Continue' + + $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\' + $NativeBaseDir = $InstallDirectory + if (!$NativeBaseDir) { + $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory + } + $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir + $InstallBin = Join-Path $NativeBaseDir 'bin' + $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1' + + # Process tools list + Write-Host "Processing $GlobalJsonFile" + If (-Not (Test-Path $GlobalJsonFile)) { + Write-Host "Unable to find '$GlobalJsonFile'" + exit 0 + } + $NativeTools = Get-Content($GlobalJsonFile) -Raw | + ConvertFrom-Json | + Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue + if ($NativeTools) { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 + } + } + else { + Write-Host 'No native tools defined in global.json' + exit 0 + } + + if ($Clean) { + exit 0 + } + if (Test-Path $InstallBin) { + Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin) + Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" + return $InstallBin + } + else { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' + exit 1 + } + exit 0 +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ + ExitWithExitCode 1 +} diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh new file mode 100755 index 0000000000000..3e6a8d6acf2f5 --- /dev/null +++ b/eng/common/init-tools-native.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' +install_directory='' +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 +global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" +declare -a native_assets + +. $scriptroot/pipeline-logging-functions.sh +. $scriptroot/native/common-library.sh + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installdirectory) + install_directory=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --donotabortonfailure) + donotabortonfailure=true + shift 1 + ;; + --donotdisplaywarnings) + donotdisplaywarnings=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --installdirectory Directory to install native toolset." + echo " This is a command-line override for the default" + echo " Install directory precedence order:" + echo " - InstallDirectory command-line override" + echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" + echo " - (default) %USERPROFILE%/.netcoreeng/native" + echo "" + echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" + echo " --force Clean and then install tools" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --baseuri Base URI for where to download native tools from" + echo " --downloadretries Number of times a download should be attempted" + echo " --retrywaittimeseconds Wait time between download attempts" + echo "" + exit 0 + ;; + esac +done + +function ReadGlobalJsonNativeTools { + # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! + if command -v jq &> /dev/null; then + + # jq: read each key/value pair under "native-tools" entry and emit: + # KEY="" VALUE="" + # followed by a null byte. + # + # bash: read line with null byte delimeter and push to array (for later `eval`uation). + + while IFS= read -rd '' line; do + native_assets+=("$line") + done < <(jq -r '. | + select(has("native-tools")) | + ."native-tools" | + keys[] as $k | + @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") + + return + fi + + # Warning: falling back to manually parsing JSON, which is not recommended. + + # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. + # It has been tested with several weird strings with escaped characters in entries (key and value) + # and results were compared with the output of jq(1) in binary representation using xxd(1); + # just before the assignment to 'native_assets' array (above and below). + + # try to capture the section under "native-tools". + if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then + return + fi + + section="${BASH_REMATCH[1]}" + + parseStarted=0 + possibleEnd=0 + escaping=0 + escaped=0 + isKey=1 + + for (( i=0; i<${#section}; i++ )); do + char="${section:$i:1}" + if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi + + if ! ((escaping)) && [[ "$char" == "\\" ]]; then + escaping=1 + elif ((escaping)) && ! ((escaped)); then + escaped=1 + fi + + if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then + parseStarted=1 + possibleEnd=0 + elif [[ "$char" == "'" ]]; then + token="$token'\\\''" + possibleEnd=0 + elif ((escaping)) || [[ "$char" != "\"" ]]; then + token="$token$char" + possibleEnd=1 + fi + + if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then + # Use printf to unescape token to match jq(1)'s @sh formatting rules. + # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. + printf -v token "'$token'" + + if ((isKey)); then + KEY="$token" + isKey=0 + else + line="KEY=$KEY VALUE=$token" + native_assets+=("$line") + isKey=1 + fi + + # reset for next token + parseStarted=0 + token= + elif ((escaping)) && ((escaped)); then + escaping=0 + escaped=0 + fi + done +} + +native_base_dir=$install_directory +if [[ -z $install_directory ]]; then + native_base_dir=$(GetNativeInstallDirectory) +fi + +install_bin="${native_base_dir}/bin" +installed_any=false + +ReadGlobalJsonNativeTools + +if [[ ${#native_assets[@]} -eq 0 ]]; then + echo "No native tools defined in global.json" + exit 0; +else + native_installer_dir="$scriptroot/native" + for index in "${!native_assets[@]}"; do + eval "${native_assets["$index"]}" + + installer_path="$native_installer_dir/install-$KEY.sh" + installer_command="$installer_path" + installer_command+=" --baseuri $base_uri" + installer_command+=" --installpath $install_bin" + installer_command+=" --version $VALUE" + echo $installer_command + + if [[ $force = true ]]; then + installer_command+=" --force" + fi + + if [[ $clean = true ]]; then + installer_command+=" --clean" + fi + + if [[ -a $installer_path ]]; then + $installer_command + if [[ $? != 0 ]]; then + if [[ $donotabortonfailure = true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + exit 1 + fi + else + $installed_any = true + fi + else + if [[ $donotabortonfailure == true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + exit 1 + fi + fi + done +fi + +if [[ $clean = true ]]; then + exit 0 +fi + +if [[ -d $install_bin ]]; then + echo "Native tools are available from $install_bin" + echo "##vso[task.prependpath]$install_bin" +else + if [[ $installed_any = true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" + exit 1 + fi +fi + +exit 0 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 new file mode 100644 index 0000000000000..92b77347d9904 --- /dev/null +++ b/eng/common/internal-feed-operations.ps1 @@ -0,0 +1,132 @@ +param( + [Parameter(Mandatory=$true)][string] $Operation, + [string] $AuthToken, + [string] $CommitSha, + [string] $RepoName, + [switch] $IsFeedPrivate +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +. $PSScriptRoot\tools.ps1 + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified +# internal builds +function SetupCredProvider { + param( + [string] $AuthToken + ) + + # Install the Cred Provider NuGet plugin + Write-Host 'Setting up Cred Provider NuGet plugin in the agent...' + Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' + + Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." + Invoke-WebRequest $url -OutFile installcredprovider.ps1 + + Write-Host 'Installing plugin...' + .\installcredprovider.ps1 -Force + + Write-Host "Deleting local copy of 'installcredprovider.ps1'..." + Remove-Item .\installcredprovider.ps1 + + if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { + Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!' + ExitWithExitCode 1 + } + else { + Write-Host 'CredProvider plugin was installed correctly!' + } + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" + + if (-Not (Test-Path -Path $nugetConfigPath)) { + Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' + ExitWithExitCode 1 + } + + $endpoints = New-Object System.Collections.ArrayList + $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} + + if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { + foreach ($stableRestoreResource in $nugetConfigPackageSources) { + $trimmedResource = ([string]$stableRestoreResource).Trim() + [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) + } + } + + if (($endpoints | Measure-Object).Count -gt 0) { + $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress + + # Create the environment variables the AzDo way + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ + 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' + 'issecret' = 'false' + } + + # We don't want sessions cached since we will be updating the endpoints quite frequently + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ + 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' + 'issecret' = 'false' + } + } + else + { + Write-Host 'No internal endpoints found in NuGet.config' + } +} + +#Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + $dotnetTempDir = Join-Path $RepoRoot "dotnet" + $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + $dotnet = "$dotnetTempDir\dotnet.exe" + $restoreProjPath = "$PSScriptRoot\restore.proj" + + Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + '' | Out-File "$restoreProjPath" + + & $dotnet restore $restoreProjPath + + Write-Host 'Arcade SDK restored!' + + if (Test-Path -Path $restoreProjPath) { + Remove-Item $restoreProjPath + } + + if (Test-Path -Path $dotnetTempDir) { + Remove-Item $dotnetTempDir -Recurse + } +} + +try { + Push-Location $PSScriptRoot + + if ($Operation -like 'setup') { + SetupCredProvider $AuthToken + } + elseif ($Operation -like 'install-restore') { + InstallDotNetSdkAndRestoreArcade + } + else { + Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!" + ExitWithExitCode 1 + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Arcade' -Message $_ + ExitWithExitCode 1 +} +finally { + Pop-Location +} diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh new file mode 100755 index 0000000000000..9378223ba0955 --- /dev/null +++ b/eng/common/internal-feed-operations.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +set -e + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. +# This should ONLY be called from identified internal builds +function SetupCredProvider { + local authToken=$1 + + # Install the Cred Provider NuGet plugin + echo "Setting up Cred Provider NuGet plugin in the agent..."... + echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" + + echo "Writing the contents of 'installcredprovider.ps1' locally..." + local installcredproviderPath="installcredprovider.sh" + if command -v curl > /dev/null; then + curl $url > "$installcredproviderPath" + else + wget -q -O "$installcredproviderPath" "$url" + fi + + echo "Installing plugin..." + . "$installcredproviderPath" + + echo "Deleting local copy of 'installcredprovider.sh'..." + rm installcredprovider.sh + + if [ ! -d "$HOME/.nuget/plugins" ]; then + Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!' + ExitWithExitCode 1 + else + echo "CredProvider plugin was installed correctly!" + fi + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + local nugetConfigPath="{$repo_root}NuGet.config" + + if [ ! "$nugetConfigPath" ]; then + Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + fi + + local endpoints='[' + local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` + local pattern="value=\"(.*)\"" + + for value in $nugetConfigPackageValues + do + if [[ $value =~ $pattern ]]; then + local endpoint="${BASH_REMATCH[1]}" + endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," + fi + done + + endpoints=${endpoints%?} + endpoints+=']' + + if [ ${#endpoints} -gt 2 ]; then + local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" + + echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" + echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" + else + echo "No internal endpoints found in NuGet.config" + fi +} + +# Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + local dotnetTempDir="$repo_root/dotnet" + local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + local restoreProjPath="$repo_root/eng/common/restore.proj" + + echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + echo "" > "$restoreProjPath" + + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + local res=`$dotnetTempDir/dotnet restore $restoreProjPath` + echo "Arcade SDK restored!" + + # Cleanup + if [ "$restoreProjPath" ]; then + rm "$restoreProjPath" + fi + + if [ "$dotnetTempDir" ]; then + rm -r $dotnetTempDir + fi +} + +source="${BASH_SOURCE[0]}" +operation='' +authToken='' +repoName='' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + --operation) + operation=$2 + shift + ;; + --authtoken) + authToken=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ "$operation" = "setup" ]; then + SetupCredProvider $authToken +elif [ "$operation" = "install-restore" ]; then + InstallDotNetSdkAndRestoreArcade +else + echo "Unknown operation '$operation'!" +fi diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props new file mode 100644 index 0000000000000..dbf99d82a5c2e --- /dev/null +++ b/eng/common/internal/Directory.Build.props @@ -0,0 +1,4 @@ + + + + diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config new file mode 100644 index 0000000000000..19d3d311b166f --- /dev/null +++ b/eng/common/internal/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj new file mode 100644 index 0000000000000..beb9c4648ea19 --- /dev/null +++ b/eng/common/internal/Tools.csproj @@ -0,0 +1,27 @@ + + + + net472 + false + false + + + + + + + + + + + https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; + + + $(RestoreSources); + https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; + + + + + + diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 new file mode 100644 index 0000000000000..f041e5ddd9589 --- /dev/null +++ b/eng/common/msbuild.ps1 @@ -0,0 +1,28 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = 'minimal', + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, + [switch] $ci, + [switch] $prepareMachine, + [switch] $excludePrereleaseVS, + [string] $msbuildEngine = $null, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs +) + +. $PSScriptRoot\tools.ps1 + +try { + if ($ci) { + $nodeReuse = $false + } + + MSBuild @extraArgs +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Build' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh new file mode 100755 index 0000000000000..20d3dad543520 --- /dev/null +++ b/eng/common/msbuild.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +verbosity='minimal' +warn_as_error=true +node_reuse=true +prepare_machine=false +extra_args='' + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --verbosity) + verbosity=$2 + shift 2 + ;; + --warnaserror) + warn_as_error=$2 + shift 2 + ;; + --nodereuse) + node_reuse=$2 + shift 2 + ;; + --ci) + ci=true + shift 1 + ;; + --preparemachine) + prepare_machine=true + shift 1 + ;; + *) + extra_args="$extra_args $1" + shift 1 + ;; + esac +done + +. "$scriptroot/tools.sh" + +if [[ "$ci" == true ]]; then + node_reuse=false +fi + +MSBuild $extra_args +ExitWithExitCode 0 diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 new file mode 100644 index 0000000000000..ca38268c44d83 --- /dev/null +++ b/eng/common/native/CommonLibrary.psm1 @@ -0,0 +1,400 @@ +<# +.SYNOPSIS +Helper module to install an archive to a directory + +.DESCRIPTION +Helper module to download and extract an archive to a specified directory + +.PARAMETER Uri +Uri of artifact to download + +.PARAMETER InstallDirectory +Directory to extract artifact contents to + +.PARAMETER Force +Force download / extraction if file or contents already exist. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds. Default = 30 + +.NOTES +Returns False if download or extraction fail, True otherwise +#> +function DownloadAndExtract { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $InstallDirectory, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 + ) + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri + + # Download native tool + $DownloadStatus = CommonLibrary\Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$Force ` + -Verbose:$Verbose + + if ($DownloadStatus -Eq $False) { + Write-Error "Download failed from $Uri" + return $False + } + + # Extract native tool + $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$Force ` + -Verbose:$Verbose + + if ($UnzipStatus -Eq $False) { + # Retry Download one more time with Force=true + $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries 1 ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$True ` + -Verbose:$Verbose + + if ($DownloadRetryStatus -Eq $False) { + Write-Error "Last attempt of download failed as well" + return $False + } + + # Retry unzip again one more time with Force=true + $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$True ` + -Verbose:$Verbose + if ($UnzipRetryStatus -Eq $False) + { + Write-Error "Last attempt of unzip failed as well" + # Clean up partial zips and extracts + if (Test-Path $TempToolPath) { + Remove-Item $TempToolPath -Force + } + if (Test-Path $InstallDirectory) { + Remove-Item $InstallDirectory -Force -Recurse + } + return $False + } + } + + return $True +} + +<# +.SYNOPSIS +Download a file, retry on failure + +.DESCRIPTION +Download specified file and retry if attempt fails + +.PARAMETER Uri +Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded + +.PARAMETER Path +Path to download or copy uri file to + +.PARAMETER Force +Overwrite existing file if present. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds Default = 30 + +#> +function Get-File { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $Path, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [switch] $Force = $False + ) + $Attempt = 0 + + if ($Force) { + if (Test-Path $Path) { + Remove-Item $Path -Force + } + } + if (Test-Path $Path) { + Write-Host "File '$Path' already exists, skipping download" + return $True + } + + $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent + if (-Not (Test-Path $DownloadDirectory)) { + New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null + } + + $TempPath = "$Path.tmp" + if (Test-Path -IsValid -Path $Uri) { + Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" + Copy-Item -Path $Uri -Destination $TempPath + Write-Verbose "Moving temporary file to '$Path'" + Move-Item -Path $TempPath -Destination $Path + return $? + } + else { + Write-Verbose "Downloading $Uri" + # Don't display the console progress UI - it's a huge perf hit + $ProgressPreference = 'SilentlyContinue' + while($Attempt -Lt $DownloadRetries) + { + try { + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath + Write-Verbose "Downloaded to temporary location '$TempPath'" + Move-Item -Path $TempPath -Destination $Path + Write-Verbose "Moved temporary file to '$Path'" + return $True + } + catch { + $Attempt++ + if ($Attempt -Lt $DownloadRetries) { + $AttemptsLeft = $DownloadRetries - $Attempt + Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $RetryWaitTimeInSeconds + } + else { + Write-Error $_ + Write-Error $_.Exception + } + } + } + } + + return $False +} + +<# +.SYNOPSIS +Generate a shim for a native tool + +.DESCRIPTION +Creates a wrapper script (shim) that passes arguments forward to native tool assembly + +.PARAMETER ShimName +The name of the shim + +.PARAMETER ShimDirectory +The directory where shims are stored + +.PARAMETER ToolFilePath +Path to file that shim forwards to + +.PARAMETER Force +Replace shim if already present. Default = False + +.NOTES +Returns $True if generating shim succeeds, $False otherwise +#> +function New-ScriptShim { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ShimName, + [Parameter(Mandatory=$True)] + [string] $ShimDirectory, + [Parameter(Mandatory=$True)] + [string] $ToolFilePath, + [Parameter(Mandatory=$True)] + [string] $BaseUri, + [switch] $Force + ) + try { + Write-Verbose "Generating '$ShimName' shim" + + if (-Not (Test-Path $ToolFilePath)){ + Write-Error "Specified tool file path '$ToolFilePath' does not exist" + return $False + } + + # WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs + # Many of the checks for installed programs expect a .exe extension for Windows tools, rather + # than a .bat or .cmd file. + # Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer + if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) { + $InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" ` + -InstallDirectory $ShimDirectory\WinShimmer ` + -Force:$Force ` + -DownloadRetries 2 ` + -RetryWaitTimeInSeconds 5 ` + -Verbose:$Verbose + } + + if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) { + Write-Host "$ShimName.exe already exists; replacing..." + Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") + } + + & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory + return $True + } + catch { + Write-Host $_ + Write-Host $_.Exception + return $False + } +} + +<# +.SYNOPSIS +Returns the machine architecture of the host machine + +.NOTES +Returns 'x64' on 64 bit machines + Returns 'x86' on 32 bit machines +#> +function Get-MachineArchitecture { + $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE + $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432 + if($ProcessorArchitecture -Eq "X86") + { + if(($ProcessorArchitectureW6432 -Eq "") -Or + ($ProcessorArchitectureW6432 -Eq "X86")) { + return "x86" + } + $ProcessorArchitecture = $ProcessorArchitectureW6432 + } + if (($ProcessorArchitecture -Eq "AMD64") -Or + ($ProcessorArchitecture -Eq "IA64") -Or + ($ProcessorArchitecture -Eq "ARM64") -Or + ($ProcessorArchitecture -Eq "LOONGARCH64")) { + return "x64" + } + return "x86" +} + +<# +.SYNOPSIS +Get the name of a temporary folder under the native install directory +#> +function Get-TempDirectory { + return Join-Path (Get-NativeInstallDirectory) "temp/" +} + +function Get-TempPathFilename { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Path + ) + $TempDir = CommonLibrary\Get-TempDirectory + $TempFilename = Split-Path $Path -leaf + $TempPath = Join-Path $TempDir $TempFilename + return $TempPath +} + +<# +.SYNOPSIS +Returns the base directory to use for native tool installation + +.NOTES +Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable +is set, or otherwise returns an install directory under the %USERPROFILE% +#> +function Get-NativeInstallDirectory { + $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY + if (!$InstallDir) { + $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/" + } + return $InstallDir +} + +<# +.SYNOPSIS +Unzip an archive + +.DESCRIPTION +Powershell module to unzip an archive to a specified directory + +.PARAMETER ZipPath (Required) +Path to archive to unzip + +.PARAMETER OutputDirectory (Required) +Output directory for archive contents + +.PARAMETER Force +Overwrite output directory contents if they already exist + +.NOTES +- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. +- Returns True if unzip operation is successful +- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory +- Returns False if unable to extract zip archive +#> +function Expand-Zip { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ZipPath, + [Parameter(Mandatory=$True)] + [string] $OutputDirectory, + [switch] $Force + ) + + Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'" + try { + if ((Test-Path $OutputDirectory) -And (-Not $Force)) { + Write-Host "Directory '$OutputDirectory' already exists, skipping extract" + return $True + } + if (Test-Path $OutputDirectory) { + Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory" + Remove-Item $OutputDirectory -Force -Recurse + if ($? -Eq $False) { + Write-Error "Unable to remove '$OutputDirectory'" + return $False + } + } + + $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" + if (Test-Path $TempOutputDirectory) { + Remove-Item $TempOutputDirectory -Force -Recurse + } + New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null + + Add-Type -assembly "system.io.compression.filesystem" + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") + if ($? -Eq $False) { + Write-Error "Unable to extract '$ZipPath'" + return $False + } + + Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory + } + catch { + Write-Host $_ + Write-Host $_.Exception + + return $False + } + return $True +} + +export-modulemember -function DownloadAndExtract +export-modulemember -function Expand-Zip +export-modulemember -function Get-File +export-modulemember -function Get-MachineArchitecture +export-modulemember -function Get-NativeInstallDirectory +export-modulemember -function Get-TempDirectory +export-modulemember -function Get-TempPathFilename +export-modulemember -function New-ScriptShim diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh new file mode 100755 index 0000000000000..080c2c283ae46 --- /dev/null +++ b/eng/common/native/common-library.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +function GetNativeInstallDirectory { + local install_dir + + if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then + install_dir=$HOME/.netcoreeng/native/ + else + install_dir=$NETCOREENG_INSTALL_DIRECTORY + fi + + echo $install_dir + return 0 +} + +function GetTempDirectory { + + echo $(GetNativeInstallDirectory)temp/ + return 0 +} + +function ExpandZip { + local zip_path=$1 + local output_directory=$2 + local force=${3:-false} + + echo "Extracting $zip_path to $output_directory" + if [[ -d $output_directory ]] && [[ $force = false ]]; then + echo "Directory '$output_directory' already exists, skipping extract" + return 0 + fi + + if [[ -d $output_directory ]]; then + echo "'Force flag enabled, but '$output_directory' exists. Removing directory" + rm -rf $output_directory + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'" + return 1 + fi + fi + + echo "Creating directory: '$output_directory'" + mkdir -p $output_directory + + echo "Extracting archive" + tar -xf $zip_path -C $output_directory + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'" + return 1 + fi + + return 0 +} + +function GetCurrentOS { + local unameOut="$(uname -s)" + case $unameOut in + Linux*) echo "Linux";; + Darwin*) echo "MacOS";; + esac + return 0 +} + +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + else + wget -q -O "$path" "$uri" --tries="$download_retries" + fi + + return $? +} + +function GetTempPathFileName { + local path=$1 + + local temp_dir=$(GetTempDirectory) + local temp_file_name=$(basename $path) + echo $temp_dir$temp_file_name + return 0 +} + +function DownloadAndExtract { + local uri=$1 + local installDir=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + local temp_tool_path=$(GetTempPathFileName $uri) + + echo "downloading to: $temp_tool_path" + + # Download file + GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'." + return 1 + fi + + # Extract File + echo "extracting from $temp_tool_path to $installDir" + ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'." + return 1 + fi + + return 0 +} + +function NewScriptShim { + local shimpath=$1 + local tool_file_path=$2 + local force=${3:-false} + + echo "Generating '$shimpath' shim" + if [[ -f $shimpath ]]; then + if [[ $force = false ]]; then + echo "File '$shimpath' already exists." >&2 + return 1 + else + rm -rf $shimpath + fi + fi + + if [[ ! -f $tool_file_path ]]; then + # try to see if the path is lower cased + tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" + if [[ ! -f $tool_file_path ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" + return 1 + fi + fi + + local shim_contents=$'#!/usr/bin/env bash\n' + shim_contents+="SHIMARGS="$'$1\n' + shim_contents+="$tool_file_path"$' $SHIMARGS\n' + + # Write shim file + echo "$shim_contents" > $shimpath + + chmod +x $shimpath + + echo "Finished generating shim '$shimpath'" + + return $? +} + diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh new file mode 100644 index 0000000000000..6d7ba15e5f2b5 --- /dev/null +++ b/eng/common/native/init-compiler.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables +# +# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! + +if [[ "$#" -lt 3 ]]; then + echo "Usage..." + echo "init-compiler.sh