Compare commits
48 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 polac24
|
c77a6341eb | ||
|
polac24
|
9a4b8eafaf | ||
|
polac24
|
0046a745f5 | ||
|
polac24
|
ac450c813e | ||
|
polac24
|
eee56b4bc6 | ||
|
polac24
|
efdb072e32 | ||
|
polac24
|
a49cfffe81 | ||
|
polac24
|
085f3ccaec | ||
|
polac24
|
0b70c83598 | ||
|
polac24
|
5d3533956c | ||
|
polac24
|
d046000ae5 | ||
|
polac24
|
233b5ad118 | ||
|
polac24
|
dca631e1b6 | ||
|
polac24
|
91a7d08a2c | ||
|
polac24
|
1898ccb87b | ||
|
polac24
|
64828903f7 | ||
|
polac24
|
f05e69617c | ||
|
polac24
|
b98496c29b | ||
|
polac24
|
1c36f66264 | ||
|
polac24
|
56fb1e13d5 | ||
|
polac24
|
84d3893ee1 | ||
|
polac24
|
c1a4a4853f | ||
|
polac24
|
74d1622a1c | ||
 aleksandergrzyb
|
9adf0c5f03 | ||
|
aleksandergrzyb
|
a454041cf3 | ||
|
aleksandergrzyb
|
1717744e95 | ||
|
polac24
|
f81b017cbe | ||
|
polac24
|
299ed72c49 | ||
 CognitiveDisson
|
815b1f9cd0 | ||
|
polac24
|
40637f1eed | ||
|
CognitiveDisson
|
72ec29533b | ||
|
CognitiveDisson
|
4e30298dae | ||
|
CognitiveDisson
|
5ba352e831 | ||
|
CognitiveDisson
|
77b5b01b33 | ||
|
CognitiveDisson
|
d0f94b1dbb | ||
|
polac24
|
a5b187912c | ||
|
polac24
|
ead2c2400d | ||
|
polac24
|
2329a01208 | ||
|
polac24
|
6f4f1ddc34 | ||
|
polac24
|
aba15fa57d | ||
|
polac24
|
bbe0b3954a | ||
|
polac24
|
19405ccbd3 | ||
|
polac24
|
bd5e5efb6a | ||
|
polac24
|
6bfd7ed5ae | ||
|
polac24
|
168a8cbd61 | ||
|
polac24
|
809c1623e4 | ||
|
polac24
|
9677a29c2b | ||
|
polac24
|
89fdf48476 |
@@ -1,65 +0,0 @@
|
||||
---
|
||||
name: ⚠️ Bug Report
|
||||
about: Something isn't working as expected
|
||||
|
||||
---
|
||||
|
||||
<!--
|
||||
PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
|
||||
-->
|
||||
|
||||
**My integration setup**
|
||||
|
||||
[ ] CocoaPods cocoapods-xcremotecache plugin
|
||||
[ ] Automatic integration using `xcprepare integrate ...`
|
||||
[ ] Manual integration
|
||||
[ ] Carthage
|
||||
|
||||
**Expected/desired behavior**
|
||||
<!-- Describe what the desired behavior would be. -->
|
||||
|
||||
**Minimal reproduction of the problem with instructions**
|
||||
<!-- Please provide the *STEPS TO REPRODUCE*. -->
|
||||
|
||||
**Producer Logs**
|
||||
<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
|
||||
|
||||
<details>
|
||||
<pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
|
||||
</details>
|
||||
|
||||
**Consumer Logs**
|
||||
<!-- Capture logs from 10 minutes: `log show --predicate 'sender BEGINSWITH "xc"' --style compact --info --debug -last 10m` -->
|
||||
|
||||
<details>
|
||||
<pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
|
||||
</details>
|
||||
|
||||
**Pods/Carthage file**
|
||||
<!-- Delete if you don't use CocoaPods or Carthage -->
|
||||
|
||||
<details>
|
||||
<pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
|
||||
</details>
|
||||
|
||||
**Environment**
|
||||
|
||||
* **XCRemoteCache:** X.Y.Z
|
||||
* **cocoapods-xcremotecache:** X.Y.Z <!-- check with `gem list cocoapods-xcremotecache` >
|
||||
* **HTTP cache server:** ... <!-- e.g. demo docker, nginx, AWS etc. >
|
||||
* **Xcode:** X.Y.Z
|
||||
|
||||
**Post build stats**
|
||||
<!--
|
||||
To capture build statistics:
|
||||
* call `xcprepare stats --reset` (or `XCRC/xcprepare stats --reset` for CocoaPods)
|
||||
* Build a project in Xcode
|
||||
* `xcprepare stats` (or `XCRC/xcprepare stats` for CocoaPods)
|
||||
-->
|
||||
|
||||
<details>
|
||||
<pre> [REPLACE THIS WITH YOUR INFORMATION] </pre>
|
||||
</details>
|
||||
|
||||
**Others**
|
||||
<!-- Anything else relevant? Operating system version, , ... -->
|
||||
@@ -1,16 +0,0 @@
|
||||
---
|
||||
name: 📕 Documentation Issue
|
||||
about: Suggestion for a change in a documentation
|
||||
|
||||
---
|
||||
|
||||
<!--
|
||||
PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
|
||||
-->
|
||||
|
||||
**A suggestion**
|
||||
<!-- Describe how could the documentation be improved. -->
|
||||
|
||||
**Which file, section, line**
|
||||
<!-- Provide a section it relates (if exist). -->
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
name: 🙏 Future Request
|
||||
about: Suggestion for an improvement, either behaviour or implementation
|
||||
|
||||
---
|
||||
|
||||
<!--
|
||||
PLEASE HELP US PROCESS GITHUB ISSUES FASTER BY PROVIDING THE FOLLOWING INFORMATION.
|
||||
-->
|
||||
|
||||
|
||||
**Expected/desired behavior**
|
||||
<!-- Describe what the desired behavior would be. -->
|
||||
|
||||
**Relevant integration setup**
|
||||
|
||||
[ ] CocoaPods cocoapods-xcremotecache plugin
|
||||
[ ] Automatic integration using `xcprepare integrate ...`
|
||||
[ ] Manual integration
|
||||
[ ] Carthage
|
||||
@@ -1,27 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
SwiftLint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: SwiftLint
|
||||
uses: norio-nomura/action-swiftlint@3.1.0
|
||||
|
||||
macOS:
|
||||
runs-on: macOS-latest
|
||||
env:
|
||||
XCODE_VERSION: ${{ '13.1' }}
|
||||
steps:
|
||||
- name: Select Xcode
|
||||
run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v1
|
||||
- name: Build and Run
|
||||
run: rake build[release]
|
||||
- name: Test
|
||||
run: rake test
|
||||
- name: E2ETests
|
||||
run: rake e2e_only
|
||||
@@ -1,72 +0,0 @@
|
||||
name: release_binaries
|
||||
on:
|
||||
release:
|
||||
types: created
|
||||
|
||||
jobs:
|
||||
macOS:
|
||||
name: Add macOS binaries to release
|
||||
runs-on: macOS-latest
|
||||
env:
|
||||
XCODE_VERSION: ${{ '13.1' }}
|
||||
steps:
|
||||
- name: Select Xcode
|
||||
run: "sudo xcode-select -s /Applications/Xcode_$XCODE_VERSION.app"
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v1
|
||||
- name: Set tag name
|
||||
run: echo "TAG_NAME=$(echo $GITHUB_REF | cut -c 11-)" >> $GITHUB_ENV
|
||||
- name: Build x86_64-apple-macosx
|
||||
run: "rake 'build[release, x86_64-apple-macosx]'"
|
||||
- name: Save x86_64 executable to be lipo'd later
|
||||
run: "mkdir -p tmp && unzip releases/XCRemoteCache.zip -d tmp/xcremotecache-x86_64"
|
||||
- name: Clean releases dir to not conflict with other archs
|
||||
run: "rm -rf releases"
|
||||
- name: Build arm64-apple-macosx
|
||||
run: "rake 'build[release, arm64-apple-macosx]'"
|
||||
- name: Save arm64 executable to be lipo'd later
|
||||
run: "mkdir -p tmp && unzip releases/XCRemoteCache.zip -d tmp/xcremotecache-arm64"
|
||||
- name: Clean releases dir to not conflict with other files to attach
|
||||
run: "rm -rf releases"
|
||||
- name: Zip x86_64-apple-macosx release
|
||||
run: "mkdir -p releases && zip -jr releases/XCRemoteCache-macOS-x86_64-$TAG_NAME.zip LICENSE README.md tmp/xcremotecache-x86_64"
|
||||
- name: Zip arm64-apple-macosx release
|
||||
run: "zip -jr releases/XCRemoteCache-macOS-arm64-$TAG_NAME.zip LICENSE README.md tmp/xcremotecache-arm64"
|
||||
- name: Lipo macOS executables
|
||||
run: "mkdir -p tmp/xcremotecache && ls tmp/xcremotecache-x86_64 | xargs -I {} lipo -create -output tmp/xcremotecache/{} tmp/xcremotecache-x86_64/{} tmp/xcremotecache-arm64/{}"
|
||||
- name: Zip x86_64-arm64-apple-macosx release
|
||||
run: "zip -jr releases/XCRemoteCache-macOS-arm64-x86_64-$TAG_NAME.zip LICENSE README.md tmp/xcremotecache"
|
||||
- name: Upload binaries to release
|
||||
uses: svenstaro/upload-release-action@v1-release
|
||||
with:
|
||||
repo_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
file: releases/*
|
||||
file_glob: true
|
||||
tag: ${{ github.ref }}
|
||||
overwrite: true
|
||||
cocoapods:
|
||||
name: Publish CocoaPods plugin
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cocoapods-plugin
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Ruby
|
||||
uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: '3.0'
|
||||
- run: bundle install
|
||||
- name: Publish to RubyGems
|
||||
run: |
|
||||
mkdir -p $HOME/.gem
|
||||
touch $HOME/.gem/credentials
|
||||
chmod 0600 $HOME/.gem/credentials
|
||||
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
||||
gem build *.gemspec
|
||||
CURRENT_VERSION=$(gem list cocoapods-xcremotecache --remote -q | sed 's/[^0-9\.]//g')
|
||||
[ -f cocoapods-xcremotecache-$CURRENT_VERSION.gem ] && echo "Version $CURRENT_VERSION already exists" || gem push *.gem
|
||||
env:
|
||||
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
.DS_Store
|
||||
/.build
|
||||
/Packages
|
||||
*.xcodeproj/
|
||||
*.xcworkspace/
|
||||
DerivedData
|
||||
/.swiftpm/
|
||||
releases
|
||||
tmp/
|
||||
.idea/
|
||||
xcuserdata
|
||||
*.gem
|
||||
Pods/
|
||||
@@ -1 +0,0 @@
|
||||
5.1
|
||||
@@ -1,45 +0,0 @@
|
||||
# Rule configurations
|
||||
--allman false
|
||||
--binarygrouping 4,8
|
||||
--commas always
|
||||
--comments indent
|
||||
--decimalgrouping 3,5
|
||||
--elseposition same-line
|
||||
--empty void
|
||||
--exponentcase lowercase
|
||||
--exponentgrouping disabled
|
||||
--fractiongrouping enabled
|
||||
--header strip
|
||||
--hexgrouping none
|
||||
--hexliteralcase lowercase
|
||||
--ifdef noindent
|
||||
--indent 4
|
||||
--indentcase false
|
||||
--xcodeindentation enabled
|
||||
--linebreaks lf
|
||||
--octalgrouping 4,8
|
||||
--patternlet inline
|
||||
--self remove
|
||||
--semicolons inline
|
||||
--stripunusedargs closure-only
|
||||
--trimwhitespace always
|
||||
--wraparguments beforefirst
|
||||
--wrapcollections beforefirst
|
||||
--closingparen balanced
|
||||
--xcodeindentation enabled
|
||||
|
||||
# Disabled rules
|
||||
--disable numberFormatting
|
||||
--disable consecutiveBlankLines
|
||||
--disable andOperator
|
||||
--disable spaceAroundOperators
|
||||
--disable redundantReturn
|
||||
--disable blankLinesAtStartOfScope
|
||||
|
||||
# Enabled rules
|
||||
|
||||
# Tool options
|
||||
--symlinks ignore
|
||||
|
||||
# Excluded directories
|
||||
--exclude Carthage,.build,DerivedData
|
||||
-129
@@ -1,129 +0,0 @@
|
||||
disabled_rules:
|
||||
- identifier_name # Does not make sense to lint for the length of identifiers.
|
||||
- type_name # Same as above.
|
||||
- empty_enum_arguments # It warns about an explicit pattern we use.
|
||||
- superfluous_disable_command # Disabled since we disable some rules pre-emptively to avoid issues in the future
|
||||
- todo # Temporarily disabled. We have too many right now hiding real issues :(
|
||||
- nesting # Does not make sense anymore since Swift 4 uses nested `CodingKeys` enums for example
|
||||
- trailing_dot_in_comments # Triggers warnings for generated file headers
|
||||
|
||||
opt_in_rules:
|
||||
- anyobject_protocol
|
||||
- attributes
|
||||
- closure_end_indentation
|
||||
- closure_spacing
|
||||
- collection_alignment
|
||||
- colon
|
||||
- contains_over_filter_count
|
||||
- contains_over_filter_is_empty
|
||||
- contains_over_first_not_nil
|
||||
- discouraged_object_literal
|
||||
- empty_collection_literal
|
||||
- empty_count
|
||||
- empty_string
|
||||
- explicit_init
|
||||
- extension_access_modifier
|
||||
- fatal_error_message
|
||||
- file_header
|
||||
- first_where
|
||||
- identical_operands
|
||||
- implicit_return
|
||||
- inert_defer
|
||||
- joined_default_parameter
|
||||
- literal_expression_end_indentation
|
||||
- legacy_hashing
|
||||
- legacy_random
|
||||
- multiline_arguments
|
||||
- multiline_literal_brackets
|
||||
- multiline_parameters
|
||||
- multiline_parameters_brackets
|
||||
- notification_center_detachment
|
||||
- number_separator
|
||||
- operator_usage_whitespace
|
||||
- overridden_super_call
|
||||
- private_action
|
||||
- prohibited_interface_builder
|
||||
- prohibited_super_call
|
||||
- redundant_nil_coalescing
|
||||
- redundant_objc_attribute
|
||||
- single_test_class
|
||||
- sorted_imports
|
||||
- static_operator
|
||||
- toggle_bool
|
||||
- trailing_comma
|
||||
- trailing_whitespace
|
||||
- unneeded_parentheses_in_closure_argument
|
||||
- vertical_parameter_alignment_on_call
|
||||
- yoda_condition
|
||||
|
||||
excluded:
|
||||
- .github/
|
||||
- .build/
|
||||
- build/
|
||||
- Carthage/
|
||||
- docs/
|
||||
- fastlane/
|
||||
- DerivedData/
|
||||
|
||||
attributes:
|
||||
always_on_same_line:
|
||||
- "@IBAction"
|
||||
- "@NSManaged"
|
||||
- "@objc"
|
||||
closure_spacing: warning
|
||||
empty_count:
|
||||
severity: warning
|
||||
implicit_return:
|
||||
included:
|
||||
- closure
|
||||
explicit_init: warning
|
||||
fatal_error_message: warning
|
||||
file_header:
|
||||
severity: warning
|
||||
forbidden_pattern: |
|
||||
\/\/
|
||||
\/\/ .*?\..*
|
||||
\/\/ .*
|
||||
\/\/
|
||||
\/\/ Created by .*? on .*\.
|
||||
\/\/ Copyright © \d{4} .*\. All rights reserved\.
|
||||
\/\/
|
||||
force_cast: warning
|
||||
force_try: warning
|
||||
implicit_getter: warning
|
||||
indentation: 4 # 4 spaces
|
||||
line_length:
|
||||
warning: 120
|
||||
error: 200
|
||||
ignores_function_declarations: true
|
||||
multiline_arguments:
|
||||
first_argument_location: next_line
|
||||
number_separator:
|
||||
minimum_length: 5 # number of digits, i.e. >= 10_000
|
||||
redundant_nil_coalescing: warning
|
||||
shorthand_operator: warning
|
||||
trailing_comma:
|
||||
mandatory_comma: true
|
||||
vertical_whitespace:
|
||||
max_empty_lines: 2
|
||||
weak_delegate: warning
|
||||
cyclomatic_complexity:
|
||||
warning: 12
|
||||
function_parameter_count:
|
||||
warning: 7
|
||||
|
||||
reporter:
|
||||
- "xcode"
|
||||
- "junit"
|
||||
|
||||
custom_rules:
|
||||
associated_values_unwrapping:
|
||||
name: "Associated Value Unwrapping"
|
||||
regex: "case let [a-zA-Z0-9]*.[a-zA-Z0-9]+\\([a-zA-Z0-9 ,]+"
|
||||
message: "Each associated value should be defined as a separate constant (i.e: .enumCase(let val1, let val2))"
|
||||
severity: warning
|
||||
trailing_dot_in_comments:
|
||||
name: "Trailing dot in comments"
|
||||
regex: '^[ ]*///?[^\n]*\.\n'
|
||||
message: "There shouldn't be trailing dot in comments"
|
||||
severity: warning
|
||||
@@ -1,3 +0,0 @@
|
||||
This project adheres to the [Open Code of Conduct][code-of-conduct]. By participating, you are expected to honor this code.
|
||||
|
||||
[code-of-conduct]: https://github.com/spotify/code-of-conduct/blob/main/code-of-conduct.md
|
||||
@@ -1,13 +0,0 @@
|
||||
# Copyright 2021 Spotify AB
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
@@ -1,70 +0,0 @@
|
||||
{
|
||||
"object": {
|
||||
"pins": [
|
||||
{
|
||||
"package": "AEXML",
|
||||
"repositoryURL": "https://github.com/tadija/AEXML",
|
||||
"state": {
|
||||
"branch": null,
|
||||
"revision": "8623e73b193386909566a9ca20203e33a09af142",
|
||||
"version": "4.5.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"package": "PathKit",
|
||||
"repositoryURL": "https://github.com/kylef/PathKit",
|
||||
"state": {
|
||||
"branch": null,
|
||||
"revision": "73f8e9dca9b7a3078cb79128217dc8f2e585a511",
|
||||
"version": "1.0.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"package": "Spectre",
|
||||
"repositoryURL": "https://github.com/kylef/Spectre.git",
|
||||
"state": {
|
||||
"branch": null,
|
||||
"revision": "f79d4ecbf8bc4e1579fbd86c3e1d652fb6876c53",
|
||||
"version": "0.9.2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"package": "swift-argument-parser",
|
||||
"repositoryURL": "https://github.com/apple/swift-argument-parser",
|
||||
"state": {
|
||||
"branch": null,
|
||||
"revision": "9f04d1ff1afbccd02279338a2c91e5f27c45e93a",
|
||||
"version": "0.0.5"
|
||||
}
|
||||
},
|
||||
{
|
||||
"package": "XcodeProj",
|
||||
"repositoryURL": "https://github.com/tuist/XcodeProj.git",
|
||||
"state": {
|
||||
"branch": null,
|
||||
"revision": "0b18c3e7a10c241323397a80cb445051f4494971",
|
||||
"version": "8.0.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"package": "Yams",
|
||||
"repositoryURL": "https://github.com/jpsim/Yams.git",
|
||||
"state": {
|
||||
"branch": null,
|
||||
"revision": "53741ba55ecca5c7149d8c9f810913ec80845c69",
|
||||
"version": "3.0.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"package": "Zip",
|
||||
"repositoryURL": "https://github.com/marmelroy/Zip.git",
|
||||
"state": {
|
||||
"branch": null,
|
||||
"revision": "80b1c3005ee25b4c7ce46c4029ac3347e8d5e37e",
|
||||
"version": "2.0.0"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"version": 1
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
// swift-tools-version:5.3
|
||||
// The swift-tools-version declares the minimum version of Swift required to build this package
|
||||
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "XCRemoteCache",
|
||||
platforms: [
|
||||
.macOS(.v10_14),
|
||||
],
|
||||
products: [
|
||||
.executable(name: "xcprebuild", targets: ["xcprebuild"]),
|
||||
],
|
||||
dependencies: [
|
||||
.package(url: "https://github.com/marmelroy/Zip.git", from: "2.0.0"),
|
||||
.package(url: "https://github.com/jpsim/Yams.git", from: "3.0.0"),
|
||||
.package(url: "https://github.com/apple/swift-argument-parser", from: "0.0.1"),
|
||||
.package(url: "https://github.com/tuist/XcodeProj.git", from: "8.0.0"),
|
||||
],
|
||||
targets: [
|
||||
.target(
|
||||
name: "XCRemoteCache",
|
||||
dependencies: ["Zip", "Yams", "XcodeProj"]
|
||||
),
|
||||
.target(
|
||||
name: "xcprebuild",
|
||||
dependencies: ["XCRemoteCache"]
|
||||
),
|
||||
.target(
|
||||
name: "xcswiftc",
|
||||
dependencies: ["XCRemoteCache"]
|
||||
),
|
||||
.target(
|
||||
name: "xclibtool",
|
||||
dependencies: ["XCRemoteCache"]
|
||||
),
|
||||
.target(
|
||||
name: "xcpostbuild",
|
||||
dependencies: ["XCRemoteCache"]
|
||||
),
|
||||
.target(
|
||||
name: "xcprepare",
|
||||
dependencies: [
|
||||
"XCRemoteCache",
|
||||
.product(name: "ArgumentParser", package: "swift-argument-parser"),
|
||||
]
|
||||
),
|
||||
.target(
|
||||
name: "xcld",
|
||||
dependencies: ["XCRemoteCache"]
|
||||
),
|
||||
.target(
|
||||
// Wrapper target that builds all binaries but does nothing in runtime
|
||||
name: "Aggregator",
|
||||
dependencies: ["xcprebuild", "xcswiftc", "xclibtool", "xcpostbuild", "xcprepare", "xcld"]
|
||||
),
|
||||
.testTarget(
|
||||
name: "XCRemoteCacheTests",
|
||||
dependencies: ["XCRemoteCache"],
|
||||
resources: [.copy("TestData")]
|
||||
),
|
||||
]
|
||||
)
|
||||
@@ -1,479 +0,0 @@
|
||||
<p align="center">
|
||||
<img src="docs/img/logo.png#gh-light-mode-only" width="75%">
|
||||
<img src="docs/img/logo-dark.png#gh-dark-mode-only" width="75%">
|
||||
</p>
|
||||
|
||||
_XCRemoteCache is a remote cache tool for Xcode projects. It reuses target artifacts generated on a remote machine, served from a simple REST server._
|
||||
|
||||
[](https://github.com/spotify/XCRemoteCache/workflows/CI/badge.svg)
|
||||
[](LICENSE)
|
||||
[](https://slackin.spotify.com)
|
||||
|
||||
- [How and Why?](#how-and-why)
|
||||
* [Accurate target input files](#accurate-target-input-files)
|
||||
+ [New file added to the target](#new-file-added-to-the-target)
|
||||
* [Debug symbols](#debug-symbols)
|
||||
* [Performance optimizations](#performance-optimizations)
|
||||
* [Focused targets](#focused-targets)
|
||||
- [How to integrate XCRemoteCache with your Xcode project?](#how-to-integrate-xcremotecache-with-your-xcode-project)
|
||||
* [1. Download XCRemoteCache](#1-download-xcremotecache)
|
||||
* [A. Automatic integration](#a-automatic-integration)
|
||||
+ [2. Create a minimal XCRemoteCache configuration](#2-create-a-minimal-xcremotecache-configuration)
|
||||
+ [3. Run automatic integration script](#3-run-automatic-integration-script)
|
||||
- [3a. Producer side](#3a-producer-side)
|
||||
- [3b. Consumer side](#3b-consumer-side)
|
||||
* [A full list of `xcprepare integrate` supported options](#a-full-list-of-xcprepare-integrate-supported-options)
|
||||
* [B. Manual integration](#b-manual-integration)
|
||||
+ [2. Configure XCRemoteCache](#2-configure-xcremotecache)
|
||||
+ [3. Call xcprepare](#3-call-xcprepare)
|
||||
+ [4. Integrate with the Xcode project](#4-integrate-with-the-xcode-project)
|
||||
+ [5. Configure LLDB source-map (Optional)](#5-configure-lldb-source-map-optional)
|
||||
+ [6. Producer mode - Artifacts generation](#6-producer-mode---artifacts-generation)
|
||||
- [6a. Configure producer mode](#6a-configure-producer-mode)
|
||||
- [6b. Fill the cache](#6b-fill-the-cache)
|
||||
- [6c. Mark commit sha](#6c-mark-commit-sha)
|
||||
- [A full list of configuration parameters:](#a-full-list-of-configuration-parameters)
|
||||
- [Backend cache server](#backend-cache-server)
|
||||
* [Sample REST cache server from a docker image](#sample-rest-cache-server-from-a-docker-image)
|
||||
* [Amazon S3 and Google Cloud Storage](#amazon-s3-and-google-cloud-storage)
|
||||
- [CocoaPods plugin](#cocoapods-plugin)
|
||||
- [Requirements](#requirements)
|
||||
- [Apple silicon support](#apple-silicon-support)
|
||||
* [Artifacts per architecture (Recommended)](#artifacts-per-architecture-recommended)
|
||||
* [Fat artifacts](#fat-artifacts)
|
||||
- [Limitations](#limitations)
|
||||
- [FAQ](#faq)
|
||||
- [Development](#development)
|
||||
- [Release](#release)
|
||||
* [Releasing CocoaPods plugin](#releasing-cocoapods-plugin)
|
||||
* [Building release package](#building-release-package)
|
||||
- [Contributing](#contributing)
|
||||
- [Code of conduct](#code-of-conduct)
|
||||
- [License](#license)
|
||||
- [Security Issues?](#security-issues)
|
||||
|
||||
## How and Why?
|
||||
|
||||
The caching mechanism is based on remote artifacts that should be generated and uploaded to the cache server for each commit on a `master` branch, preferably as a part of CI/CD step. Xcode products are not portable between different Xcode versions, each XCRemoteCache artifact is linked with a specific Xcode build number that generated it. To support multiple Xcode versions, artifacts generation should happen for each Xcode version.
|
||||
|
||||
The artifact reuse flow is as follows: XCRemoteCache performs a target precheck (aka prebuild) and if a fingerprint for local sources matches the one computed on a generation side, several compilation steps wrappers (e.g. `xcswiftc`, `xccc`, `xclibtool`) mock corresponding compilation step(s) and linking (or archiving) moves the cached build artifact to the expected location.
|
||||
|
||||
> Multiple commits that have the same target sources reuse artifact package on a remote server.
|
||||
|
||||
### Accurate target input files
|
||||
|
||||
Finding a precise list of input and dependency files is a non-trivial task as Xcode heavily relies on implicit target dependencies. It means that Xcode is trying to use required dependencies from provided search paths and looks up `DerivedData`'s product dir. To find a narrow list of files to compute fingerprint hash, XCRemoteCache fetches a `meta.json` file from a server which contains remotelly generated fingerprint hash and a list of input files that should constitute a fingerprint. That list of input files was produced during the artifact generation process, based on `.d` output from `clang` and `swift` compilers.
|
||||
|
||||
Before building a project in Xcode, XCRemoteCache needs to find the best git commit sha for which artifacts will be used. This happens as a part of the `xcprepare` execution, which should be called after each merge or switching a branch. `xcprepare` finds a list of 10 most recent common sha with the remote repo branch using git's first-parent strategy and selects the newest one for which all artifacts have been uploaded.
|
||||
|
||||
_The generation side is responsible to call `xcprepare mark` subcommand after each successful build. Marking process creates an empty marker file on a remote cache server with a given format: `#{commmmitSha}-#{TargetName}-#{Configuration}-#{Platform}-#{XcodeBuildNumber}-#{ContextBuildSettings}-#{SchemaID}.json`._
|
||||
|
||||
`xcprepare` makes `HEAD` requests for all identified shas, picks the newest one for which a marker file exists remotely, and saves it in the text form to the `arc.rc` file. That file informs the prebuild phase which meta file should be fetched to get a list of target dependency files.
|
||||
|
||||
#### New file added to the target
|
||||
|
||||
Considering in the hash fingerprint only a list of previously observed files can give invalid results if a build contains a new source file as it isn't considered in the hash.
|
||||
|
||||
For a new `.swift` file in a swift-only target, `xcswiftc` automatically recognizes that case and forces local compilation of the entire target. For Objective-C or mixed targets, fallbacking to the local compilation is more difficult as some previous invocations (either `xccc` or `xcswiftc`) could already be finished with no-operation. To mitigate that, each wrapper appends invocation call to a side file (`history.compile`) just in case some other process would need to compile the entire target locally. If that happens, compilation of the newly added file acquires a target-wide lock that stops other wrapper invocations, executes already mocked steps one by one to backfill already skipped compilation steps.
|
||||
|
||||
### Debug symbols
|
||||
|
||||
Binaries built with "debug symbols: enabled" embed source file absolute paths so compilation products cannot be directly ported between two machines with different source roots. Otherwise, LLDB debugger is not able to correlate a set of currently executing machine instructions with a local file that produced it. To mitigate that, XCRemoteCache recommends adding a custom C and Swift debug flags `prefix-map` for all XCRemoteCache builds. These flags ensure that all binaries, generated locally and downloaded from a remote server, have the same debug symbols absolute paths which are translated to an actual local path at the beginning of the LLDB session.
|
||||
|
||||
### Performance optimizations
|
||||
|
||||
XCRemoteCache involves several optimization techniques:
|
||||
|
||||
* Local HTTP cache stores all responses from the remote server at `~/Library/Caches/XCRemoteCache/`
|
||||
* Prebuild and postbuild steps leverage Xcode's discovered dependency file to avoid recomputing fingerprint hashes if none of the input files has changed
|
||||
* A wrapper for the `clang` compilation is a C program, generated and compiled during the `xcprepare` step. It is called many times to compile each `*.(m|c)` file and accessing a disk to read a configuration would introduce a significant slowdown, especially if a project contains a lot of Objective-C files. As a remedy, `xcprepare` reads the XCRemoteCache configuration only once and embeds all configurable fields directly into the `xccc` binary
|
||||
* `arc.rc`, generated by `xcprepare`, gets file modification equal to the commit date if refers. `arc.rc` is included in the discovered dependency file, touching it in the `xcprepare` would automatically invalidate previous XCRemoteCache prebuild step and force redundant fingerprint checks. By syncing the `mdate` with a git commit, Xcode avoids prebuild steps unless the remote cache commit has changed
|
||||
* If a target cache miss happens, XCRemoteCache disables cache for that target until a commit sha in `arc.rc` changes. That bypasses a fingerprint computation for incremental builds
|
||||
|
||||
### Focused targets
|
||||
|
||||
If a list of targets that can have dirty sources is limited, XCRemoteCache can be configured with focused targets, specified in `.rcinfo`.
|
||||
|
||||
By default, all targets are focused and these compare local fingerprint with one available remotely and fallbacks to the local compilation if it doesn't match. Non-focused targets, called 'thin' targets, always use cached artifacts what eliminates a fingerprint computation. Thin targets should contain only a single compilation file with `thin_target_mock_filename`, e.g. `standin.swift` or `standin.m`.
|
||||
|
||||
## How to integrate XCRemoteCache with your Xcode project?
|
||||
|
||||
To enable XCRemoteCache in the existing `.xcodeproj` you need to add extra build settings and build phases to targets that you want to cache.
|
||||
|
||||
You can do that in an automatic way, using the XCRemoteCache-provided integration command, or manually modify your Xcode project.
|
||||
|
||||
### 1. Download XCRemoteCache
|
||||
|
||||
From the Github [Releases page](https://github.com/spotify/XCRemoteCache/releases), download the XCRemoteCache bundle zip. Unzip the bundle to a directory next to your `.xcodeproj`.
|
||||
|
||||
_The following steps will assume the bundle has been unzipped to `xcremotecache` dir, placed next to the `.xcodeproj`._
|
||||
|
||||
### A. Automatic integration
|
||||
|
||||
#### 2. Create a minimal XCRemoteCache configuration
|
||||
|
||||
Create `.rcinfo` yaml file next to the `.xcodeproj` with a minimum set of configuration entries, like:
|
||||
```yaml
|
||||
primary_repo: https://yourRepo.git
|
||||
cache_addresses:
|
||||
- https://xcremotecacheserver.com
|
||||
```
|
||||
|
||||
#### 3. Run automatic integration script
|
||||
|
||||
##### 3a. Producer side
|
||||
|
||||
Execute a command that modifies `<yourProject.xcodeproj>`:
|
||||
```bash
|
||||
xcremotecache/xcprepare integrate --input <yourProject.xcodeproj> --mode producer --final-producer-target <YourMainTarget>
|
||||
```
|
||||
|
||||
##### 3b. Consumer side
|
||||
|
||||
Execute a command that modifies `<yourProject.xcodeproj>`:
|
||||
```bash
|
||||
xcremotecache/xcprepare integrate --input <yourProject.xcodeproj> --mode consumer
|
||||
```
|
||||
|
||||
###### A full list of `xcprepare integrate` supported options
|
||||
|
||||
| Argument | Description | Default | Required |
|
||||
| ------------- | ------------- | ------------- | ------------- |
|
||||
| `--input` | .xcodeproj location | N/A | ✅ |
|
||||
| `--mode` | mode. Supported values: `consumer`, `producer`, `producer-fast`(experimental) | N/A | ✅ |
|
||||
| `--targets-include` | comma-separated list of targets to integrate XCRemoteCache. | `""` | ⬜️ |
|
||||
| `--targets-exclude` | comma-separated list of targets to not integrate XCRemoteCache. Takes priority over --targets-include. | `""` | ⬜️ |
|
||||
| `--configurations-include` | comma-separated list of configurations to integrate XCRemoteCache. | `""` | ⬜️ |
|
||||
| `--configurations-exclude` | comma-separated list of configurations to not integrate XCRemoteCache. Takes priority over --configurations-include. | `Release` | ⬜️ |
|
||||
| `--final-producer-target` | [Producer only] The final target that generates cache artifacts. Once this targets is finished, no other targets are allowed to upload artifacts to the remote server for a given sha, configuration and platform context. | `nil` | ⬜️ |
|
||||
| `--consumer-eligible-configurations` | [Consumer only] comma-separated list of configurations that need to have all artifacts uploaded to the remote site before using given sha. | `Debug` | ⬜️ |
|
||||
| `--consumer-eligible-platforms` | [Consumer only] comma-separated list of platforms that need to have all artifacts uploaded to the remote site before using given sha | `iphonesimulator` | ⬜️ |
|
||||
| `--lldb-init` | LLDBInit mode. Appends to .lldbinit a command required for debugging. Supported values: 'none' (do not append to .lldbinit), 'user' (append to ~/.lldbinit) | `user` | ⬜️ |
|
||||
| `--fake-src-root` | An arbitrary source location shared between producers and consumers. Should be unique for a project. | `/xxxxxxxxxx` | ⬜️ |
|
||||
| `--output` | Save the project with integrated XCRemoteCache to a separate location. | N/A | ⬜️ |
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### B. Manual integration
|
||||
|
||||
#### 2. Configure XCRemoteCache
|
||||
|
||||
Create yaml configuration file `.rcinfo`, next to the `.xcodeproj`, with your full XCRemoteCache configuration, according to [the parameters list](#a-full-list-of-configuration-parameters) e.g.:
|
||||
|
||||
```yaml
|
||||
primary_repo: https://yourRepo.git
|
||||
cache_addresses:
|
||||
- https://xcremotecacheserver.com
|
||||
repo_root: "."
|
||||
remote_commit_file: arc.rc
|
||||
xccc_file: xcremotecache/xccc
|
||||
```
|
||||
|
||||
#### 3. Call xcprepare
|
||||
|
||||
Execute `xcprepare --configuration #Configuration# --platform #platform#` command after each merge or rebase with the primary branch. Otherwise, the remote cache artifacts may be outdated and final hit rate may be poor.
|
||||
|
||||
The `xcprepare` application saves `arc.rc` file on a disk and prints a summary to the standard output. The printed `recommended_remote_address` is just a recommendation which cache remote server use. It is up to the integration tooling to decide if it makes sense. If so, the project's `.rcinfo` should define that value as `recommended_remote_address` parameter.
|
||||
|
||||
Example:
|
||||
|
||||
```shell
|
||||
$ xcremotecache/xcprepare --configuration Debug --platform iphonesimulator
|
||||
result: true
|
||||
commit: aabbccc00
|
||||
age: 0
|
||||
recommended_remote_address: https://xcremotecacheserver.com
|
||||
```
|
||||
|
||||
#### 4. Integrate with the Xcode project
|
||||
|
||||
Configure Xcode targets that **should use** XCRemoteCache:
|
||||
|
||||
1. Override Build Settings:
|
||||
* `CC` - `xccc_file` from your `.rcinfo` configuration (e.g. `xcremotecache/xccc`)
|
||||
* `SWIFT_EXEC` - location of `xcprepare` (e.g. `xcremotecache/xcswiftc`)
|
||||
* `LIBTOOL` - location of `xclibtool` (e.g. `xcremotecache/xclibtool`)
|
||||
* `LD` - location of `xcld` (e.g. `xcremotecache/xcld`)
|
||||
* `XCRC_PLATFORM_PREFERRED_ARCH` - `$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)`
|
||||
|
||||
<details>
|
||||
<summary>Screenshot</summary>
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
2. Add a `Prebuild` build phase (before compilation):
|
||||
* command: `"$SCRIPT_INPUT_FILE_0"`
|
||||
* input files: location of `xcprebuild` (e.g. `xcremotecache/xcprebuild`)
|
||||
* output files:
|
||||
* `$(TARGET_TEMP_DIR)/rc.enabled`
|
||||
* `$(DWARF_DSYM_FOLDER_PATH)/$(DWARF_DSYM_FILE_NAME)`
|
||||
* discovery dependency file: `$(TARGET_TEMP_DIR)/prebuild.d`
|
||||
3. Add `Postbuild` build phase (after compilation):
|
||||
* command: `"$SCRIPT_INPUT_FILE_0"`
|
||||
* input files: location of `xcpostbuild` command (e.g. `xcremotecache/xcpostbuild`)
|
||||
* output files:
|
||||
* `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH).swiftmodule.md5`
|
||||
* `$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/$(XCRC_PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5`
|
||||
* discovery dependency file: `$(TARGET_TEMP_DIR)/postbuild.d`
|
||||
|
||||
<details>
|
||||
<summary>Screenshot</summary>
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
#### 5. Configure LLDB source-map (Optional)
|
||||
|
||||
Rewriting source-map is required to support debugging and hit breakpoints, see [Debug symbols](#debug-symbols).
|
||||
|
||||
1. Ooverride the following Build Settings for **all targets**:
|
||||
* `XCRC_SRCROOT` - `/xxxxxxxxxx` (or any other arbitrary string for your project)
|
||||
* add `-debug-prefix-map $(SRCROOT)=$(XCRC_SRCROOT)` to `OTHER_SWIFT_FLAGS`. _If it doesn't exist, define it as `$(inherited) -debug-prefix-map $(SRCROOT)=$(XCRC_SRCROOT)`_
|
||||
* add `-fdebug-prefix-map=$(SRCROOT)=$(XCRC_SRCROOT)` to `OTHER_CFLAGS`. _If it doesn't exists, define it as `$(inherited) -fdebug-prefix-map=$(SRCROOT)=$(XCRC_SRCROOT)`_
|
||||
2. Add `settings set target.source-map /xxxxxxxxxx /Users/account/src/PathToTheProject` to `~/.lldbinit` on end machine that builds a project with XCRemoteCache
|
||||
|
||||
> `XCRC_SRCROOT` arbitrary path should be project-exclusive to avoid clashing.
|
||||
|
||||
_Tip: In some rare cases, Xcode caches `~/.lldbinit` content so make sure to restart Xcode after the modification._
|
||||
|
||||
#### 6. Producer mode - Artifacts generation
|
||||
|
||||
XCRemoteCache can operate in two main modes: `consumer` (default) tries to reuse artifacts available on the remote server and `producer` is used to generate all artifacts - it builds all targets locally and uploads meta and artifact files to the remote cache server.
|
||||
|
||||
##### 6a. Configure producer mode
|
||||
|
||||
To enable the `producer` mode, configure it directly in the `.rcinfo` file.
|
||||
|
||||
> Optionally, you can define `extra_configuration_file` in a `.rcinfo` with a path to the other yaml file that will override the default configuration in `.rcinfo`. That approach can be useful if you want to track main `.rcinfo` and keep your local configuration out of git.
|
||||
|
||||
##### 6b. Fill the cache
|
||||
|
||||
Build the project from Xcode or using `xcodebuild`
|
||||
|
||||
##### 6c. Mark commit sha
|
||||
|
||||
Once all artifacts have been uploaded, "mark a build" using `xcprepare mark` command:
|
||||
|
||||
```shell
|
||||
$ xcremotecache/xcprepare mark --configuration Debug --platform iphonesimulator
|
||||
```
|
||||
|
||||
That command creates an empty file on a remote server which informs that for given sha, configuration, platform, Xcode versions etc. all artifacts are available.
|
||||
|
||||
_Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, `xclibtool` wrappers become no-op, so it is recommended to not add them for the `producer` mode._
|
||||
|
||||
## A full list of configuration parameters:
|
||||
|
||||
| Property | Description | Default | Required |
|
||||
| ------------- | ------------- | ------------- | ------------- |
|
||||
| `mode` | build mode. Possible values: `consumer`, `producer` | `consumer` | ⬜️ |
|
||||
| `cache_addresses` | Addresses of all remote cache replicas. _Required to be a non-empty array of strings_ | N/A | ✅ |
|
||||
| `recommended_cache_address` | Address of the best remote cache to use in the consumer mode. If not specified, the first item in `cache_addresses` will be used | N/A | ⬜️ |
|
||||
| `cache_health_path` | Probe request path to the `cache_addresses` (relative to a path in `cache_addresses`) that determines the best cache to use | `nginx-health` | ⬜️ |
|
||||
| `cache_health_path_probe_count` | Number of `cacheAddresses` probe requests | `3` | ⬜️ |
|
||||
| `remote_commit_file` | Filepath to the file with the remote commit sha | `build/remote-cache/arc.rc` | ⬜️ |
|
||||
| `xccc_file` | Path to the xccc wrapper | `build/bin/xccc` | ⬜️ |
|
||||
| `prebuild_discovery_path` | Path, relative to `$TARGET_TEMP_DIR`, that specifies prebuild discovery .d file | `prebuild.d` | ⬜️ |
|
||||
| `postbuild_discovery_path` | Path, relative to `$TARGET_TEMP_DIR`, that specifies postbuild discovery .d file | `postbuild.d` | ⬜️ |
|
||||
| `mode_marker_path` | Path, relative to `$TARGET_TEMP_DIR`, of a maker file to enable or disable the remote cache for a given target. Includes a list of all allowed input files to use remote cache | `rc.enabled` | ⬜️ |
|
||||
| `clang_command` | Command for a standard C compilation fallback | `clang` | ⬜️ |
|
||||
| `swiftc_command` | Command for a standard Swift compilation fallback | `swiftc` | ⬜️ |
|
||||
| `primary_repo` | Address of the primary git repository that produces cache artifacts (case-sensitive) | N/A | ✅ |
|
||||
| `primary_branch` | The main (primary) branch on the `primary_repo` that produces cache artifacts | `master` | ⬜️ |
|
||||
| `repo_root` | The path to the git repo root | `"."` | ⬜️ |
|
||||
| `cache_commit_history` | Number of historical git commits to look for cache artifacts | `10` | ⬜️ |
|
||||
| `source_root` | Source root of the Xcode project | `""` | ⬜️ |
|
||||
| `fingerprint_override_extension` | Fingerprint override extension (sample override `Module.swiftmodule/x86_64.swiftmodule.md5`) | `md5` | ⬜️ |
|
||||
| `extra_configuration_file` | Configuration file that overrides project configuration | `user.rcinfo` | ⬜️ |
|
||||
| `publishing_sha` | Custom commit sha to publish artifact (producer only) | `nil` | ⬜️ |
|
||||
| `artifact_maximum_age` | Maximum age in days HTTP response should be locally cached before being evicted | `30` | ⬜️ |
|
||||
| `custom_fingerprint_envs` | Extra ENV keys that should be convoluted into the environment fingerprint | `[]` | ⬜️ |
|
||||
| `stats_dir` | Directory where all XCRemoteCache statistics (e.g. counters) are stored | `~/.xccache` | ⬜️ |
|
||||
| `download_retries` | Number of retries for download requests | `0` | ⬜️ |
|
||||
| `upload_retries` | Number of retries for upload requests | `3` | ⬜️ |
|
||||
| `request_custom_headers` | Dictionary of extra HTTP headers for all remote server requests | `[]` | ⬜️ |
|
||||
| `thin_target_mock_filename` | Filename (without an extension) of the compilation input file that is used as a fake compilation for the forced-cached target (aka thin target) | `standin` | ⬜️ |
|
||||
| `focused_targets` | A list of all targets that are not thinned. If empty, all targets are meant to be non-thin | `[]` | ⬜️ |
|
||||
| `disable_http_cache ` | Disable cache for http requests to fetch metadata and download artifacts | `false` | ⬜️ |
|
||||
| `compilation_history_file ` | Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be executed if a target switches to local compilation. Example: A new `.swift` file invalidates remote artifact and triggers local compilation. When that happens, all previously skipped clang build steps need to be eventually called locally - this file lists all these commands. | `history.compile` | ⬜️ |
|
||||
| `timeout_response_data_chunks_interval ` | Timeout for remote response data interval (in seconds). If an interval between data chunks is longer than a timeout, a request fails. | `20` | ⬜️ |
|
||||
| `turn_off_remote_cache_on_first_timeout ` | If true, any observed request timeout switches off remote cache for all targets | `false` | ⬜️ |
|
||||
| `product_files_extensions_with_content_override ` | List of all extensions that should carry over source fingerprints. Extensions of all product files that contain non-deterministic content (absolute paths, timestamp, etc) should be included. | `["swiftmodule"]` | ⬜️ |
|
||||
| `thinning_enabled ` | If true, support for thin projects is enabled | `false` | ⬜️ |
|
||||
| `thinning_target_module_name ` | Module name of a target that works as a helper for thinned targets | `"ThinningRemoteCacheModule"` | ⬜️ |
|
||||
| `prettify_meta_files` | A Boolean value that opts-in pretty JSON formatting for meta files | `false` | ⬜️ |
|
||||
| `aws_secret_key` | Secret key for AWS V4 Signature Authorization. If this is set to a non-empty String - an Authentication Header will be added based on this and the other `aws_*` parameters.| `""` | ⬜️ |
|
||||
| `aws_access_key` | Access key for AWS V4 Signature Authorization. | `""` | ⬜️ |
|
||||
| `aws_region` | Region for AWS V4 Signature Authorization. E.g. `eu`. | `""` | ⬜️ |
|
||||
| `aws_service` | Service for AWS V4 Signature Authorization. E.g. `storage`. | `""` | ⬜️ |
|
||||
| `out_of_band_mappings` | A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies. Useful if a project refers files out of repo root, either compilation files or precompiled dependencies. Keys represent generic replacement and values are substrings that should be replaced. Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]` `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`). Warning: remapping order is not-deterministic so avoid remappings with multiple matchings. | `[:]` | ⬜️ |
|
||||
| `disable_certificate_verification` | A Boolean value that opts-in SSL certificate validation is disabled | `false` | ⬜️ |
|
||||
| `disable_vfs_overlay` | A feature flag to disable virtual file system overlay support (temporary) | `false` | ⬜️ |
|
||||
| `custom_rewrite_envs` | A list of extra ENVs that should be used as placeholders in the dependency list. ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process. | `[]` | ⬜️ |
|
||||
|
||||
## Backend cache server
|
||||
|
||||
As a cache server, XCRemoteCache may use any REST server that supports PUT, GET and HEAD methods.
|
||||
|
||||
For the development phase, you can try the simplest cache server available as a docker image in [backend-example](backend-example). For the production environment, it is recommended to configure a reliable, fast server, preferrably located in a close proximity to developer's machines.
|
||||
|
||||
Out-of-the-box, XCRemoteCache supports V4 Signature Authorization used by Amazon's S3 and Google's GCS. Altenatively, if your server has a customized authentication procedure, you can add extra HTTP request headers with `request_custom_headers` configuration property.
|
||||
|
||||
### Sample REST cache server from a docker image
|
||||
|
||||
To run a local instance of a server, use a snippet which exposes a cache endpoint under `http://localhost:8080/cache`:
|
||||
|
||||
```bash
|
||||
docker build -t xcremotecache-demo-server backend-example
|
||||
docker run -it --rm -d -p 8080:8080 --name xcremotecache xcremotecache-demo-server
|
||||
```
|
||||
|
||||
As the docker image saves all files in a container non-persistent storage, to reset cache's content, just restart it:
|
||||
```bash
|
||||
# stop the container
|
||||
docker kill xcremotecache
|
||||
# run a new instance of the image
|
||||
docker run -it --rm -d -p 8080:8080 --name xcremotecache xcremotecache-demo-server
|
||||
```
|
||||
|
||||
To review all files stored in the cache server, navigate to the container's cache root directory:
|
||||
```bash
|
||||
docker exec -w /tmp/cache -it xcremotecache /bin/bash
|
||||
```
|
||||
|
||||
### Amazon S3 and Google Cloud Storage
|
||||
|
||||
XCRemoteCache supports Amazon S3 and Google Cloud Storage buckets to be used as cache servers using the Amazon v4 Signature Authorization.
|
||||
|
||||
To set it up use the configuration parameters `aws_secret_key`, `aws_access_key`, `aws_region`, and `aws_service` in the `.rcinfo` file. Specify the URL to the bucket in cache-addresses field in the same file.
|
||||
|
||||
Example
|
||||
```yaml
|
||||
...
|
||||
cache_addresses:
|
||||
- https://bucketname.s3.eu-central-1.amazonaws.com/
|
||||
aws_secret_key: <SECRET_KEY>
|
||||
aws_access_key: <ACCESS_KEY>
|
||||
aws_region: eu-central-1
|
||||
aws_service: s3
|
||||
...
|
||||
```
|
||||
|
||||
Retention Policy: Buckets usually have a retention policy option which ensures objects are retained for a certain amount of time and won't be modified or deleted. Keep this option short or disable it to avoid errors in case multiple builds are done consecutively on the producer side for the same configuration.
|
||||
|
||||
## CocoaPods plugin
|
||||
|
||||
Head over to our [cocoapods-plugin](cocoapods-plugin/README.md) docs to see how to integrate XCRemoteCache in your CocoaPods project.
|
||||
|
||||
## Apple silicon support
|
||||
|
||||
### Artifacts per architecture (Recommended)
|
||||
|
||||
_If all of your machines (both producer and all consumers have the same architecture, either Intel or Apple Silicon), you don't have to do anything._
|
||||
|
||||
XCRemoteCache supports building artifacts for Apple silicon consumers. Is it recommended to build separately for `x86_64` and `arm64` architectures to have single-architecture artifacts that do not require downloading irrelevant binaries. Here are required steps if you want to support both Intel and Apple silicon consumers.
|
||||
|
||||
* Building for a simulator on a producer: run a first build for `x86_64`, clean a build and build again for `arm64`, e.g.:
|
||||
```
|
||||
xcodebuild ARCHS=x86_64 ONLY_ACTIVE_ARCH=NO build ...
|
||||
xcodebuild clean
|
||||
xcodebuild ARCHS=arm64 ONLY_ACTIVE_ARCH=NO build ...
|
||||
```
|
||||
|
||||
### Fat artifacts
|
||||
|
||||
If you prefer to generate far artifacts (with both Intel and Apple silicon binaries), you can disable "Build Archive Architecture Only" on a producer side, e.g.
|
||||
|
||||
```
|
||||
xcodebuild ONLY_ACTIVE_ARCH=NO build ...
|
||||
```
|
||||
|
||||
Note: This setup is not recommended and may not be supported in future XCRemoteCache releases.
|
||||
|
||||
## Requirements
|
||||
|
||||
* The repo under `git` version control
|
||||
* Xcode 11.4+
|
||||
* Xcode New Build System
|
||||
* Current Xcode location set by `xcode-select`
|
||||
* Using the default Xcode Toolchain
|
||||
* Recommended: multi-targets Xcode project
|
||||
* Recommended: do not use fast-forward PR strategy (use merge or squash instead)
|
||||
* Recommended: avoid `DWARF with dSYM File` "Debug Information Format" build setting. Use `DWARF` instead
|
||||
|
||||
## Limitations
|
||||
|
||||
* Swift Package Manager (SPM) dependencies are not supported. _Because SPM does not allow customizing Build Settings, XCRemoteCache cannot specify `clang` and `swiftc` wrappers that control if the local compilation should be skipped (cache hit) or not (cache miss)_
|
||||
* Filenames with `_vers.c` suffix are reserved and cannot be used as a source file
|
||||
* All compilation files should be referenced via the git repo root. Referencing `/AbsolutePath/someOther.swift` or `../../someOther.swift` that resolve to the location outside of the git repo root is prohibited.
|
||||
|
||||
## FAQ
|
||||
|
||||
Follow the [FAQ](docs/FAQ.md) page.
|
||||
|
||||
## Development
|
||||
|
||||
Follow the [Development](docs/Development.md) guide. It has all the information on how to get started.
|
||||
|
||||
## Release
|
||||
|
||||
To release a version, in [Releases](https://github.com/spotify/XCRemoteCache/releases) draft a new release with `v0.3.0{-rc0}` tag format.
|
||||
Packages with binaries will be automatically uploaded to the GitHub [Releases](https://github.com/spotify/XCRemoteCache/releases) page.
|
||||
|
||||
### Releasing CocoaPods plugin
|
||||
|
||||
Bump a gem version defined in [gem_version.rb](cocoapods-plugin/lib/cocoapods-xcremotecache/gem_version.rb) and create a new release described above.
|
||||
|
||||
A plugin is automatically uploaded to [RubyGems](https://rubygems.org/gems/cocoapods-xcremotecache) if a given version doesn't exist yet.
|
||||
|
||||
### Building release package
|
||||
|
||||
To build a release zip package for a single platform (e.g. `x86_64-apple-macosx`, `arm64-apple-macosx`), call:
|
||||
|
||||
```shell
|
||||
rake 'build[release, x86_64-apple-macosx]'
|
||||
```
|
||||
|
||||
The zip package will be generated at `releases/XCRemoteCache.zip`.
|
||||
|
||||
## Support
|
||||
|
||||
Create a [new issue](https://github.com/spotify/XCRemoteCache/issues/new) with as many details as possible.
|
||||
|
||||
Reach us at the `#xcremotecache` channel in [Slack](https://slackin.spotify.com/).
|
||||
|
||||
## Contributing
|
||||
|
||||
We feel that a welcoming community is important and we ask that you follow Spotify's
|
||||
[Open Source Code of Conduct](https://github.com/spotify/code-of-conduct/blob/master/code-of-conduct.md)
|
||||
in all interactions with the community.
|
||||
|
||||
## Code of conduct
|
||||
|
||||
This project adheres to the [Open Code of Conduct](https://github.com/spotify/code-of-conduct/blob/master/code-of-conduct.md). By participating, you are expected to honor this code.
|
||||
|
||||
## License
|
||||
|
||||
```
|
||||
Copyright 2021 Spotify AB
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
```
|
||||
|
||||
## Security Issues?
|
||||
|
||||
Please report sensitive security issues via Spotify's bug-bounty program (https://hackerone.com/spotify) rather than GitHub.
|
||||
@@ -1,130 +0,0 @@
|
||||
# encoding: utf-8
|
||||
require_relative 'tasks/e2e'
|
||||
|
||||
################################
|
||||
# Rake configuration
|
||||
################################
|
||||
|
||||
# Paths
|
||||
DERIVED_DATA_DIR = File.join('.build').freeze
|
||||
RELEASES_ROOT_DIR = File.join('releases').freeze
|
||||
|
||||
EXECUTABLE_NAME = 'XCRemoteCache'
|
||||
EXECUTABLE_NAMES = ['xclibtool', 'xcpostbuild', 'xcprebuild', 'xcprepare', 'xcswiftc', 'xcld']
|
||||
PROJECT_NAME = 'XCRemoteCache'
|
||||
|
||||
SWIFTLINT_ENABLED = true
|
||||
SWIFTFORMAT_ENABLED = true
|
||||
|
||||
################################
|
||||
# Tasks
|
||||
################################
|
||||
|
||||
task :prepare do
|
||||
Dir.mkdir(DERIVED_DATA_DIR) unless File.exists?(DERIVED_DATA_DIR)
|
||||
end
|
||||
|
||||
desc 'lint'
|
||||
task :lint => [:prepare] do
|
||||
puts 'Run linting'
|
||||
|
||||
system("swiftformat --lint --config .swiftformat --cache ignore .") or abort "swiftformat failure" if SWIFTFORMAT_ENABLED
|
||||
system("swiftlint lint --config .swiftlint.yml --strict") or abort "swiftlint failure" if SWIFTLINT_ENABLED
|
||||
end
|
||||
|
||||
task :autocorrect => [:prepare] do
|
||||
puts 'Run autocorrect'
|
||||
|
||||
system("swiftformat --config .swiftformat --cache ignore .") or abort "swiftformat failure" if SWIFTFORMAT_ENABLED
|
||||
system("swiftlint autocorrect --config .swiftlint.yml") or abort "swiftlint failure" if SWIFTLINT_ENABLED
|
||||
end
|
||||
|
||||
desc 'build package artifacts'
|
||||
task :build, [:configuration, :arch, :sdks, :is_archive] do |task, args|
|
||||
# Set task defaults
|
||||
args.with_defaults(:configuration => 'debug', :sdks => ['macos'])
|
||||
|
||||
unless args.configuration == 'Debug'.downcase || args.configuration == 'Release'.downcase
|
||||
fail("Unsupported configuration. Valid values: ['Debug', 'Release']. Found '#{args.configuration}''")
|
||||
end
|
||||
|
||||
# Clean data generated by SPM
|
||||
# FIXME: dangerous recursive rm
|
||||
system("rm -rf #{DERIVED_DATA_DIR} > /dev/null 2>&1")
|
||||
|
||||
# Build
|
||||
build_paths = []
|
||||
args.sdks.each do |sdk|
|
||||
spm_build(args.configuration, args.arch)
|
||||
|
||||
# Path of the executable looks like: `.build/(debug|release)/XCRemoteCache`
|
||||
build_path_base = File.join(DERIVED_DATA_DIR, args.configuration)
|
||||
sdk_build_paths = EXECUTABLE_NAMES.map {|e| File.join(build_path_base, e)}
|
||||
|
||||
build_paths.push(sdk_build_paths)
|
||||
end
|
||||
|
||||
puts "Build products: #{build_paths}"
|
||||
|
||||
if args.configuration == 'Release'.downcase
|
||||
puts "Creating release zip"
|
||||
create_release_zip(build_paths[0])
|
||||
end
|
||||
end
|
||||
|
||||
desc 'run tests with SPM'
|
||||
task :test do
|
||||
# Running tests
|
||||
spm_test()
|
||||
end
|
||||
|
||||
desc 'build and run E2E tests'
|
||||
task :e2e => [:build, :e2e_only]
|
||||
|
||||
desc 'run E2E tests without building the XCRemoteCache binary'
|
||||
task :e2e_only => ['e2e:run']
|
||||
|
||||
################################
|
||||
# Helper functions
|
||||
################################
|
||||
|
||||
def spm_build(configuration, arch)
|
||||
spm_cmd = "swift build "\
|
||||
"-c #{configuration} "\
|
||||
"#{arch.nil? ? "" : "--triple #{arch}"} "
|
||||
system(spm_cmd) or abort "Build failure"
|
||||
end
|
||||
|
||||
def bash(command)
|
||||
system "bash -c \"#{command}\""
|
||||
end
|
||||
|
||||
def spm_test()
|
||||
tests_output_file = File.join(DERIVED_DATA_DIR, 'tests.log')
|
||||
# Redirect error stream with to a file and pass to the second stream output
|
||||
spm_cmd = "swift test --enable-code-coverage 2> >(tee #{tests_output_file})"
|
||||
test_succeeded = bash(spm_cmd)
|
||||
|
||||
abort "Test failure" unless test_succeeded
|
||||
end
|
||||
|
||||
def create_release_zip(build_paths)
|
||||
release_dir = RELEASES_ROOT_DIR
|
||||
|
||||
# Create and move files into the release directory
|
||||
mkdir_p release_dir
|
||||
build_paths.each {|p|
|
||||
cp_r p, release_dir
|
||||
}
|
||||
|
||||
output_artifact_basename = "#{PROJECT_NAME}.zip"
|
||||
|
||||
Dir.chdir(release_dir) do
|
||||
# -X: no extras (uid, gid, file times, ...)
|
||||
# -x: exclude .DS_Store
|
||||
# -r: recursive
|
||||
system("zip -X -x '*.DS_Store' -r #{output_artifact_basename} .") or abort "zip failure"
|
||||
# List contents of zip file
|
||||
system("unzip -l #{output_artifact_basename}") or abort "unzip failure"
|
||||
end
|
||||
end
|
||||
@@ -1,20 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
// Aggregator is an empty target to rebuild all executables
|
||||
@@ -1,113 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Locally generated artifact
|
||||
struct Artifact {
|
||||
/// Unique identifier of an artifact
|
||||
let id: String
|
||||
/// Location of the generated artifact package
|
||||
let package: URL
|
||||
/// Location of the generated meta file
|
||||
let meta: URL
|
||||
}
|
||||
|
||||
/// Creates a local artifact that contains all products generated in the building process
|
||||
protocol ArtifactCreator {
|
||||
func createArtifact(artifactKey: String, meta: MainArtifactMeta) throws -> Artifact
|
||||
}
|
||||
|
||||
class BuildArtifactCreator: ArtifactSwiftProductsBuilderImpl, ArtifactCreator {
|
||||
private let buildDir: URL
|
||||
private let tempDir: URL
|
||||
private let executablePath: String
|
||||
private let moduleName: String?
|
||||
private let modulesFolderPath: String
|
||||
private let dSYMPath: URL
|
||||
private let metaWriter: MetaWriter
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(
|
||||
buildDir: URL,
|
||||
tempDir: URL,
|
||||
executablePath: String,
|
||||
moduleName: String?,
|
||||
modulesFolderPath: String,
|
||||
dSYMPath: URL,
|
||||
metaWriter: MetaWriter,
|
||||
fileManager: FileManager
|
||||
) {
|
||||
self.buildDir = buildDir
|
||||
self.modulesFolderPath = modulesFolderPath
|
||||
self.tempDir = tempDir
|
||||
self.executablePath = executablePath
|
||||
self.moduleName = moduleName
|
||||
self.fileManager = fileManager
|
||||
self.dSYMPath = dSYMPath
|
||||
self.metaWriter = metaWriter
|
||||
super.init(workingDir: tempDir, moduleName: moduleName, fileManager: fileManager)
|
||||
}
|
||||
|
||||
func createArtifact(artifactKey: String, meta: MainArtifactMeta) throws -> Artifact {
|
||||
let zipWorkingDir = buildingArtifactLocation()
|
||||
|
||||
let binary = buildDir.appendingPathComponent(executablePath)
|
||||
var zipPaths = [binary]
|
||||
let swiftArtifacts = try prepareSwiftArtifacts(tempDir: zipWorkingDir)
|
||||
zipPaths.append(contentsOf: swiftArtifacts)
|
||||
let dynamicLibraryArtifacts = try prepareDynamicLibraryArtifacts()
|
||||
zipPaths.append(contentsOf: dynamicLibraryArtifacts)
|
||||
|
||||
let creator = ZipArtifactCreator(
|
||||
workingDir: zipWorkingDir,
|
||||
metaWriter: metaWriter,
|
||||
fileManager: fileManager
|
||||
)
|
||||
return try creator.createArtifact(zipContent: zipPaths, artifactKey: artifactKey, meta: meta)
|
||||
}
|
||||
|
||||
/// Prepare optional swift products: .swiftmodule, .swiftdoc, -Swift.h
|
||||
/// - Parameter tempDir: Temp location to organize file hierarchy in the artifact
|
||||
/// - returns: URLs to include into the artifact package
|
||||
fileprivate func prepareSwiftArtifacts(tempDir: URL) throws -> [URL] {
|
||||
var artifacts: [URL] = []
|
||||
|
||||
// Add optional directory with generated ObjC headers
|
||||
let generatedObjCURL = buildingArtifactObjCHeadersLocation()
|
||||
if fileManager.fileExists(atPath: generatedObjCURL.path) {
|
||||
artifacts.append(generatedObjCURL)
|
||||
}
|
||||
|
||||
// Add optional directory with generated .swiftmodule files
|
||||
let generatedSwiftModuleURL = buildingArtifactSwiftModulesLocation()
|
||||
if fileManager.fileExists(atPath: generatedSwiftModuleURL.path) {
|
||||
artifacts.append(generatedSwiftModuleURL)
|
||||
}
|
||||
return artifacts
|
||||
}
|
||||
|
||||
/// Returns a list of extra files to bundle, related to the dynamic library (if present)
|
||||
fileprivate func prepareDynamicLibraryArtifacts() throws -> [URL] {
|
||||
if fileManager.fileExists(atPath: dSYMPath.path) {
|
||||
return [dSYMPath]
|
||||
}
|
||||
return []
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Zip
|
||||
|
||||
enum ArtifactOrganizerError: Error {
|
||||
case invalidLocation(URL)
|
||||
}
|
||||
|
||||
enum ArtifactOrganizerLocationPreparationResult: Equatable {
|
||||
/// Artifact already exists at the artifactDir
|
||||
case artifactExists(artifactDir: URL)
|
||||
/// Ready to download the artifact into artifact location
|
||||
case preparedForArtifact(artifact: URL)
|
||||
}
|
||||
|
||||
/// Prepares .zip artifact for the local operations
|
||||
protocol ArtifactOrganizer {
|
||||
/// Prepares the location for the artifact unzipping
|
||||
/// - Parameter fileKey: artifact fileKey that corresponds to the zip filename on the remote cache server
|
||||
func prepareArtifactLocationFor(fileKey: String) throws -> ArtifactOrganizerLocationPreparationResult
|
||||
/// Unzips the zip artifact at the URL
|
||||
func prepare(artifact: URL) throws -> URL
|
||||
/// Activates the artifact - to all other xc* applications use it (links the directory to the "active" location)
|
||||
func activate(extractedArtifact: URL) throws
|
||||
/// Returns local location of the artifact to use in cached scenario (aka active artifact)
|
||||
func getActiveArtifactLocation() -> URL
|
||||
/// Returns a fileKey of the current active artifact
|
||||
func getActiveArtifactFilekey() throws -> String
|
||||
}
|
||||
|
||||
class ZipArtifactOrganizer: ArtifactOrganizer {
|
||||
private let cacheDir: URL
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(targetTempDir: URL, fileManager: FileManager) {
|
||||
cacheDir = targetTempDir.appendingPathComponent("xccache")
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
private func getArtifactLocation(for fileKey: String) -> URL {
|
||||
return cacheDir.appendingPathComponent(fileKey)
|
||||
}
|
||||
|
||||
func getActiveArtifactLocation() -> URL {
|
||||
return cacheDir.appendingPathComponent("active")
|
||||
}
|
||||
|
||||
func getActiveArtifactFilekey() throws -> String {
|
||||
let activeLocation = getActiveArtifactLocation()
|
||||
// Context specific fingerprint is used as a name of an active directory symlink. That ensures that
|
||||
// aritfacts do not mix up with each other but also gives a chance here to quickly get a fingerprint string
|
||||
let localArtifactLocation = try fileManager.spt_followSymbolicLink(activeLocation)
|
||||
return localArtifactLocation.lastPathComponent
|
||||
}
|
||||
|
||||
func prepareArtifactLocationFor(fileKey: String) throws -> ArtifactOrganizerLocationPreparationResult {
|
||||
let artifactDirURL = getArtifactLocation(for: fileKey)
|
||||
let artifactPackageURL = artifactDirURL.appendingPathExtension("zip")
|
||||
|
||||
if fileManager.fileExists(atPath: artifactDirURL.path) {
|
||||
return .artifactExists(artifactDir: artifactDirURL)
|
||||
}
|
||||
try createParentLocation(for: artifactPackageURL)
|
||||
return .preparedForArtifact(artifact: artifactPackageURL)
|
||||
}
|
||||
|
||||
|
||||
func prepare(artifact: URL) throws -> URL {
|
||||
let destinationURL = artifact.deletingPathExtension()
|
||||
guard fileManager.fileExists(atPath: destinationURL.path) == false else {
|
||||
infoLog("Skipping artifact, already existing at \(destinationURL)")
|
||||
return destinationURL
|
||||
}
|
||||
// Uzipping to a temp file first to never leave the uncompleted zip in the final location
|
||||
// when the command was interrupted (internal crash or `kill -9` signal)
|
||||
let tempDestination = destinationURL.appendingPathExtension("tmp")
|
||||
try Zip.unzipFile(artifact, destination: tempDestination, overwrite: true, password: nil)
|
||||
try fileManager.moveItem(at: tempDestination, to: destinationURL)
|
||||
return destinationURL
|
||||
}
|
||||
|
||||
func activate(extractedArtifact: URL) throws {
|
||||
let activeLocationURL = getActiveArtifactLocation()
|
||||
try fileManager.spt_forceSymbolicLink(at: activeLocationURL, withDestinationURL: extractedArtifact)
|
||||
}
|
||||
|
||||
private func createParentLocation(for file: URL) throws {
|
||||
let directoryURL = file.deletingLastPathComponent()
|
||||
var isDir: ObjCBool = false
|
||||
if fileManager.fileExists(atPath: directoryURL.path, isDirectory: &isDir) {
|
||||
guard isDir.boolValue else {
|
||||
errorLog("Invalid Artifact parent location at: \(directoryURL.description)")
|
||||
throw ArtifactOrganizerError.invalidLocation(directoryURL)
|
||||
}
|
||||
} else {
|
||||
try fileManager.createDirectory(at: directoryURL, withIntermediateDirectories: true, attributes: nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
|
||||
/// Plugin that can extend the artifact creation phase
|
||||
protocol ArtifactCreatorPlugin {
|
||||
|
||||
/// Optional remapper that replaces dependencies paths
|
||||
/// Useful when a plugin modifies underlying file locations in the compilation step
|
||||
var customPathsRemapper: DependenciesRemapper? { get }
|
||||
|
||||
/// Gives a chance to append extra keys to the meta type that will be uploaded to the cache server
|
||||
/// - Parameter meta: existing meta
|
||||
/// - Returns: extra dictionary that should be appended to the meta's extraKeys field
|
||||
func extraMetaKeys(_ meta: MainArtifactMeta) throws -> [String: String]
|
||||
|
||||
/// Optional artifacts that should be uploaded to the remote server
|
||||
/// - Parameter main: main artifact that has been uploaded to the remote cache server
|
||||
/// - Returns: list of artifacts that should be uploaded
|
||||
func artifactToUpload(main: MainArtifactMeta) throws -> [Artifact]
|
||||
}
|
||||
|
||||
|
||||
/// Plugin that manages addons to the artifact consumption phase (in the prebuild phase)
|
||||
protocol ArtifactConsumerPrebuildPlugin {
|
||||
/// Called when the artifact preparation phase happens. Intended to download all companion artifacts uploaded
|
||||
/// from the `artifactToUpload` returned items
|
||||
/// - Parameter meta: main artifact meta
|
||||
func run(meta: MainArtifactMeta) throws
|
||||
}
|
||||
|
||||
/// Plugin that manages addons to the artifact consumption phase (in the postbuild phase)
|
||||
protocol ArtifactConsumerPostbuildPlugin {
|
||||
/// Called after the target has been reused from cache
|
||||
/// - Parameter meta: main artifact meta
|
||||
func run(meta: MainArtifactMeta) throws
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum ArtifactSwiftProductsBuilderError: Error {
|
||||
/// Thrown when trying to include generated ObjC header to a non-module target
|
||||
case populatingObjCHeaderForNonModule
|
||||
/// Throws when trying to include non-existing ObjC header
|
||||
case populatingNonExistingObjCHeader
|
||||
/// Missing generated swiftmodule-related file (e.f. .swiftmodule or .swiftdoc)
|
||||
case missingGeneratedModuleFile(path: String)
|
||||
}
|
||||
|
||||
/// A builder to prepare artifact Swift-generated products in a single location, ready to zip into an artifact archive
|
||||
protocol ArtifactSwiftProductsBuilder {
|
||||
/// Location where all files expected to be bundled to the product should be placed
|
||||
func buildingArtifactLocation() -> URL
|
||||
/// Location where all generated ObjC headers should be placed in order to be bundled into the artifact product
|
||||
/// - Returns: location URL to put ObjC headers
|
||||
func buildingArtifactObjCHeadersLocation() -> URL
|
||||
/// Moves generated ObjC header to the artifact "working" location
|
||||
/// - Parameter arch: architecture of the build
|
||||
/// - Parameter headerURL: file to include as an ObjC header
|
||||
func includeObjCHeaderToTheArtifact(arch: String, headerURL: URL) throws
|
||||
/// Moves generated .swift{module|doc} products to the artifact "working" location
|
||||
/// - Parameter arch: architecture of the build
|
||||
/// - Parameter moduleURL: generated .swift{module|doc|..} file
|
||||
func includeModuleDefinitionsToTheArtifact(arch: String, moduleURL: URL) throws
|
||||
}
|
||||
|
||||
/// Default Builder implementation for a Swift module compilation step
|
||||
/// * all files are stored in #{workingDir}/xccache/produced
|
||||
/// * all module ObjC headers are stored in
|
||||
/// # {workingDir}/xccache/produced/include/#{moduleName} (if `moduleName` is defined)
|
||||
class ArtifactSwiftProductsBuilderImpl: ArtifactSwiftProductsBuilder {
|
||||
|
||||
private let workingDir: URL
|
||||
private let moduleName: String?
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(workingDir: URL, moduleName: String?, fileManager: FileManager) {
|
||||
self.workingDir = workingDir
|
||||
self.moduleName = moduleName
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func buildingArtifactLocation() -> URL {
|
||||
return workingDir.appendingPathComponent("xccache").appendingPathComponent("produced")
|
||||
}
|
||||
|
||||
func buildingArtifactObjCHeadersLocation() -> URL {
|
||||
return buildingArtifactLocation().appendingPathComponent("include")
|
||||
}
|
||||
|
||||
func buildingArtifactSwiftModulesLocation() -> URL {
|
||||
return buildingArtifactLocation().appendingPathComponent("swiftmodule")
|
||||
}
|
||||
|
||||
func includeObjCHeaderToTheArtifact(arch: String, headerURL: URL) throws {
|
||||
guard let module = moduleName else {
|
||||
throw ArtifactSwiftProductsBuilderError.populatingObjCHeaderForNonModule
|
||||
}
|
||||
let zipObjCDir = buildingArtifactObjCHeadersLocation()
|
||||
// Embed the ObjC header to the include/arch/module_name directory (XCRemoteCache arbitrary format)
|
||||
let moduleObjCURL = zipObjCDir.appendingPathComponent(arch).appendingPathComponent(module)
|
||||
|
||||
let objCHeaderFilename = headerURL.lastPathComponent
|
||||
let headerArtifactURL = moduleObjCURL.appendingPathComponent(objCHeaderFilename)
|
||||
// Product module dir may not exist, even if the `moduleName` is present
|
||||
guard fileManager.fileExists(atPath: headerURL.path) else {
|
||||
throw ArtifactSwiftProductsBuilderError.populatingNonExistingObjCHeader
|
||||
}
|
||||
try fileManager.createDirectory(at: moduleObjCURL, withIntermediateDirectories: true, attributes: nil)
|
||||
try fileManager.spt_forceLinkItem(at: headerURL, to: headerArtifactURL)
|
||||
}
|
||||
|
||||
func includeModuleDefinitionsToTheArtifact(arch: String, moduleURL: URL) throws {
|
||||
let zipModuleDir = buildingArtifactSwiftModulesLocation()
|
||||
// Embed the swiftmodule|doc to the swiftmodule/arch/ directory (XCRemoteCache arbitrary format)
|
||||
let artifactModuleURL = zipModuleDir.appendingPathComponent(arch)
|
||||
|
||||
let moduleURLDir = moduleURL.deletingLastPathComponent()
|
||||
let swiftModuleFilename = moduleURL.deletingPathExtension().lastPathComponent
|
||||
let swiftArtifactModuleBase = moduleURLDir.appendingPathComponent(swiftModuleFilename)
|
||||
let filesToInclude: [URL] = try SwiftmoduleFileExtension.SwiftmoduleExtensions.compactMap { ext, type in
|
||||
let file = swiftArtifactModuleBase.appendingPathExtension(ext.rawValue)
|
||||
guard fileManager.fileExists(atPath: file.path) else {
|
||||
if case .required = type {
|
||||
throw ArtifactSwiftProductsBuilderError.missingGeneratedModuleFile(path: file.path)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return file
|
||||
}
|
||||
// Product module dir may not exist, even if the `moduleName` is present
|
||||
try fileManager.createDirectory(at: artifactModuleURL, withIntermediateDirectories: true, attributes: nil)
|
||||
for fileToInclude in filesToInclude {
|
||||
let filename = fileToInclude.lastPathComponent
|
||||
let artifactLocation = artifactModuleURL.appendingPathComponent(filename)
|
||||
try fileManager.spt_forceLinkItem(at: fileToInclude, to: artifactLocation)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum SwiftmoduleFileExtensionType {
|
||||
case required
|
||||
case optional
|
||||
}
|
||||
|
||||
// Type of the file that constitutes a full modulemap package
|
||||
// RawValue corresponds to the file extension
|
||||
enum SwiftmoduleFileExtension: String {
|
||||
case swiftmodule
|
||||
case swiftdoc
|
||||
case swiftsourceinfo
|
||||
case swiftinterface
|
||||
}
|
||||
|
||||
extension SwiftmoduleFileExtension {
|
||||
/// List of all swiftmodule extensions that should be copied to the artifact
|
||||
static let SwiftmoduleExtensions: [SwiftmoduleFileExtension: SwiftmoduleFileExtensionType] = [
|
||||
.swiftmodule: .required,
|
||||
.swiftdoc: .required,
|
||||
.swiftsourceinfo: .optional,
|
||||
.swiftinterface: .optional,
|
||||
]
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Zip
|
||||
|
||||
class ZipArtifactCreator {
|
||||
/// Location where zip file should be generated
|
||||
private let workingDir: URL
|
||||
private let metaWriter: MetaWriter
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(workingDir: URL, metaWriter: MetaWriter, fileManager: FileManager) {
|
||||
self.workingDir = workingDir
|
||||
self.metaWriter = metaWriter
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func createArtifact<T: Meta>(zipContent: [URL], artifactKey: String, meta: T) throws -> Artifact {
|
||||
let zipURL = workingDir.appendingPathComponent("\(artifactKey).zip")
|
||||
try fileManager.createDirectory(at: workingDir, withIntermediateDirectories: true, attributes: nil)
|
||||
// Include meta json to the artifact
|
||||
let metaURL = try metaWriter.write(meta, locationDir: workingDir)
|
||||
let zipPaths = zipContent + [metaURL]
|
||||
|
||||
try Zip.zipFiles(paths: zipPaths, zipFilePath: zipURL, password: nil, progress: nil)
|
||||
return Artifact(id: artifactKey, package: zipURL, meta: metaURL)
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Represents a mode that libtool was called
|
||||
public enum XCLibtoolMode {
|
||||
/// Creating a static library (ar format) from a set of .o input files
|
||||
case createLibrary(output: String, filelist: String, dependencyInfo: String)
|
||||
/// Creating a universal library (multiple-architectures) from a set of input .a static libraries
|
||||
case createUniversalBinary(output: String, inputs: [String])
|
||||
}
|
||||
|
||||
public class XCLibtool {
|
||||
private let logic: XCLibtoolLogic
|
||||
|
||||
/// Intializer that depending on the argument mode, creates different libtool logic (kind of abstract factory)
|
||||
/// - Parameter mode: libtool mode to setup
|
||||
/// - Throws: XCLibtoolLogic specific errors if the mode arguments are invalid or inconsistent
|
||||
public init(_ mode: XCLibtoolMode) throws {
|
||||
switch mode {
|
||||
case .createLibrary(let output, let filelist, let dependencyInfo):
|
||||
logic = XCCreateBinary(
|
||||
output: output,
|
||||
filelist: filelist,
|
||||
dependencyInfo: dependencyInfo,
|
||||
fallbackCommand: "libtool",
|
||||
stepDescription: "Libtool"
|
||||
)
|
||||
case .createUniversalBinary(let output, let inputs):
|
||||
logic = try XCLibtoolCreateUniversalBinary(output: output, inputs: inputs)
|
||||
}
|
||||
}
|
||||
|
||||
/// Executes the libtool logic
|
||||
public func run() {
|
||||
logic.run()
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum XCLibtoolCreateUniversalBinaryError: Error {
|
||||
/// Missing ar libraries that should constitute an universal build
|
||||
case missingInputLibrary
|
||||
}
|
||||
|
||||
/// Wrapper for `libtool` call for creating an universal binary
|
||||
class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
|
||||
private let output: URL
|
||||
private let tempDir: URL
|
||||
private let firstInputURL: URL
|
||||
|
||||
init(output: String, inputs: [String]) throws {
|
||||
self.output = URL(fileURLWithPath: output)
|
||||
guard let firstInput = inputs.first else {
|
||||
throw XCLibtoolCreateUniversalBinaryError.missingInputLibrary
|
||||
}
|
||||
let firstInputURL = URL(fileURLWithPath: firstInput)
|
||||
// inputs are place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/Binary/$TARGET_NAME.a
|
||||
// TODO: find better (stable) technique to determine `$TARGET_TEMP_DIR`
|
||||
tempDir = firstInputURL
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
self.firstInputURL = firstInputURL
|
||||
}
|
||||
|
||||
func run() {
|
||||
// check if RC is enabled. if so, take any input .a and copy to the output location
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
do {
|
||||
let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
|
||||
config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
|
||||
.readConfiguration()
|
||||
} catch {
|
||||
errorLog("Libtool initialization failed with error: \(error). Fallbacking to libtool")
|
||||
fallbackToDefault()
|
||||
}
|
||||
|
||||
let markerURL = tempDir.appendingPathComponent(config.modeMarkerPath)
|
||||
do {
|
||||
let markerReader = FileMarkerReader(markerURL, fileManager: fileManager)
|
||||
guard markerReader.canRead() else {
|
||||
fallbackToDefault()
|
||||
}
|
||||
|
||||
// Remote cache artifact stores a final library from DerivedData/Products location
|
||||
// (an universal binary here)
|
||||
// Fot a target where universal binary is used as a product, an output from single-architecture `xclibtool`
|
||||
// already is a universal library (the one from artifact package)
|
||||
// Link any of input libraries (here first) to the final output location because xclibtool flow ensures
|
||||
// that these are already an universal binary
|
||||
try fileManager.spt_forceLinkItem(at: firstInputURL, to: output)
|
||||
} catch {
|
||||
errorLog("Libtool failed with error: \(error). Fallbacking to libtool")
|
||||
do {
|
||||
try fileManager.removeItem(at: markerURL)
|
||||
fallbackToDefault()
|
||||
} catch {
|
||||
exit(1, "FATAL: libtool failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func fallbackToDefault() -> Never {
|
||||
let args = ProcessInfo().arguments
|
||||
let command = "libtool"
|
||||
let paramList = [command] + args.dropFirst()
|
||||
let cargs = paramList.map { strdup($0) } + [nil]
|
||||
execvp(command, cargs)
|
||||
|
||||
/// C-function `execv` returns only when the command fails
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
/// XCLibtool wrapper logic that executes the libtool logic
|
||||
protocol XCLibtoolLogic {
|
||||
/// Executes xclibtool mocked logic or fallbacks to the libtool execution
|
||||
func run()
|
||||
}
|
||||
|
||||
extension XCCreateBinary: XCLibtoolLogic {}
|
||||
@@ -1,26 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Errors thrown from Plugins
|
||||
enum PluginError: Error {
|
||||
/// The error is severe and the command should fail immediately
|
||||
case unrecoverableError(Error)
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum ArtifactInspectorError: Error {
|
||||
/// The unzipped artifact is malformed. Is misses *.swiftmodule file in "swiftmodule/{{arch}}" directory
|
||||
case missingSwiftmoduleFileInArtifact(artifact: URL)
|
||||
}
|
||||
|
||||
// Inspects the unzipped artifact
|
||||
protocol ArtifactInspector {
|
||||
/// Enumerates all files in an artifact and finds out which should be moved to the builtProductsDir
|
||||
/// - Parameter artifact: location of the unzipped artifact
|
||||
/// - Returns: all files/dirs to move to builtProductsDir
|
||||
func findBinaryProducts(fromArtifact artifact: URL) throws -> [URL]
|
||||
/// Inspects unzipped artifact file structure to recognize the name of a module name
|
||||
func recognizeModuleName(fromArtifact artifact: URL, arch: String) throws -> String?
|
||||
}
|
||||
|
||||
class DefaultArtifactInspector: ArtifactInspector {
|
||||
private let dirAccessor: DirAccessor
|
||||
/// Name of a directory in an artifact that stores swiftmodules files
|
||||
private static let ArtifactSwiftmoduleDir = "swiftmodule"
|
||||
/// Swiftmodule file extension in an artifact
|
||||
private static let SwiftmoduleFileExtension = "swiftmodule"
|
||||
/// Extensions of files that should be considered as binaries
|
||||
// TODO: Supporting only libraries for now. Consider other formats like frameworks or dsyms
|
||||
private static let BinaryProductsExtensions = ["a"]
|
||||
|
||||
init(dirAccessor: DirAccessor) {
|
||||
self.dirAccessor = dirAccessor
|
||||
}
|
||||
|
||||
func findBinaryProducts(fromArtifact artifact: URL) throws -> [URL] {
|
||||
let artifactItems = try dirAccessor.items(at: artifact)
|
||||
return artifactItems.filter { Self.BinaryProductsExtensions.contains($0.pathExtension) }
|
||||
}
|
||||
|
||||
func recognizeModuleName(fromArtifact artifact: URL, arch: String) throws -> String? {
|
||||
let swiftmodulesDir = artifact
|
||||
.appendingPathComponent(Self.ArtifactSwiftmoduleDir)
|
||||
.appendingPathComponent(arch)
|
||||
guard case .dir = try dirAccessor.itemType(atPath: swiftmodulesDir.path) else {
|
||||
// This target doesn't contain any swiftmodule (e.g. ObjC target)
|
||||
return nil
|
||||
}
|
||||
// All files have basename of a modulename
|
||||
let moduleFiles = try dirAccessor.items(at: swiftmodulesDir)
|
||||
// Find a first *.swiftmodule file's basename - the "swiftmodule/{{arch}}" directory contains
|
||||
// {{moduleName}}.swiftc{module|doc} files
|
||||
let swiftmoduleFile = moduleFiles.first(where: { $0.pathExtension == Self.SwiftmoduleFileExtension })
|
||||
guard swiftmoduleFile != nil else {
|
||||
throw ArtifactInspectorError.missingSwiftmoduleFileInArtifact(artifact: artifact)
|
||||
}
|
||||
return swiftmoduleFile.map { $0.deletingPathExtension().lastPathComponent }
|
||||
}
|
||||
}
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Factory to create `ArtifactOrganizer`
|
||||
protocol ThinningConsumerArtifactsOrganizerFactory {
|
||||
/// Builds artifacts aggregator that oranizes artifacts in a dedicated target temp dir
|
||||
/// - Parameter targetTempDir: location where should the organizer organize the artifact ($TARGET_TEMP_DIR)
|
||||
func build(targetTempDir: URL) -> ArtifactOrganizer
|
||||
}
|
||||
|
||||
class ThinningConsumerZipArtifactsOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory {
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(fileManager: FileManager) {
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func build(targetTempDir: URL) -> ArtifactOrganizer {
|
||||
ZipArtifactOrganizer(targetTempDir: targetTempDir, fileManager: fileManager)
|
||||
}
|
||||
}
|
||||
-109
@@ -1,109 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Factory that builds Swift products organizer (that can place all files in the final Derived Data location)
|
||||
/// for a specific module
|
||||
protocol ThinningConsumerSwiftProductsOrganizerFactory {
|
||||
/// Builds products organizer that produces swift products (swiftmodule, swiftdoc) for a given module
|
||||
/// - Parameters:
|
||||
/// - architecture: .swiftmodule architecture to generate
|
||||
/// (it can be an extended arch, like "x86_64-apple-ios-simulator")
|
||||
/// - targetName: name of the target to generate
|
||||
/// - moduleName: name of the module to generate
|
||||
/// - artifactLocation: location of the unzipped artifact
|
||||
func build(
|
||||
architecture: String,
|
||||
targetName: String,
|
||||
moduleName: String,
|
||||
artifactLocation: URL
|
||||
) -> SwiftProductsOrganizer
|
||||
}
|
||||
|
||||
/// Factory that syncs swiftc products from the the unzipped artifacts and uses product generator that
|
||||
/// employs hard linking to place files in the desired location
|
||||
class ThinningConsumerUnzippedArtifactSwiftProductsOrganizerFactory: ThinningConsumerSwiftProductsOrganizerFactory {
|
||||
/// The base architecture - that current build is compiling. Equals $(ARCHS)
|
||||
private let arch: String
|
||||
private let productsLocationProvider: SwiftProductsLocationProvider
|
||||
private let fingerprintSyncer: FingerprintSyncer
|
||||
private let diskCopier: DiskCopier
|
||||
|
||||
/// Default initializer
|
||||
/// - Parameters:
|
||||
/// - arch: current architecture that the target is building for
|
||||
/// - productsLocationProvider: a provider that provides swift products final location
|
||||
/// - fingerprintSyncer: a syncer to decorate swift products with a figerprint override
|
||||
/// - fileManager: FileManager
|
||||
init(
|
||||
arch: String,
|
||||
productsLocationProvider: SwiftProductsLocationProvider,
|
||||
fingerprintSyncer: FingerprintSyncer,
|
||||
diskCopier: DiskCopier
|
||||
) {
|
||||
self.arch = arch
|
||||
self.productsLocationProvider = productsLocationProvider
|
||||
self.fingerprintSyncer = fingerprintSyncer
|
||||
self.diskCopier = diskCopier
|
||||
}
|
||||
|
||||
/// Generates a swift products generator for a specific architecture and moduleName
|
||||
/// - Parameters:
|
||||
/// - architecture: .swiftmodule architecture to generate
|
||||
/// (it can be an extended arch, like "x86_64-apple-ios-simulator")
|
||||
/// - targetName: target name to generate
|
||||
/// - moduleName: swiftmodule name
|
||||
private func buildGenerator(architecture: String, targetName: String, moduleName: String) -> SwiftcProductsGenerator {
|
||||
let modulePathOutput = productsLocationProvider.swiftmoduleFileLocation(
|
||||
moduleName: moduleName,
|
||||
architecture: architecture
|
||||
)
|
||||
let objcHeaderOutput = productsLocationProvider.objcHeaderLocation(
|
||||
targetName: targetName,
|
||||
moduleName: moduleName
|
||||
)
|
||||
|
||||
return ThinningDiskSwiftcProductsGenerator(
|
||||
modulePathOutput: modulePathOutput,
|
||||
objcHeaderOutput: objcHeaderOutput,
|
||||
diskCopier: diskCopier
|
||||
)
|
||||
}
|
||||
|
||||
func build(
|
||||
architecture: String,
|
||||
targetName: String,
|
||||
moduleName: String,
|
||||
artifactLocation: URL
|
||||
) -> SwiftProductsOrganizer {
|
||||
let productGenerator = buildGenerator(
|
||||
architecture: architecture,
|
||||
targetName: targetName,
|
||||
moduleName: moduleName
|
||||
)
|
||||
return UnzippedArtifactSwiftProductsOrganizer(
|
||||
arch: arch,
|
||||
moduleName: moduleName,
|
||||
artifactLocation: artifactLocation,
|
||||
productsGenerator: productGenerator,
|
||||
fingerprintSyncer: fingerprintSyncer
|
||||
)
|
||||
}
|
||||
}
|
||||
-104
@@ -1,104 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum WorkerResult {
|
||||
case successes
|
||||
case errors([Error])
|
||||
}
|
||||
|
||||
/// Worker that manages executing blocks
|
||||
protocol Worker {
|
||||
/// Adding an action to run in parallel
|
||||
/// - Parameter action: action to perform
|
||||
func appendAction(_ action: @escaping () throws -> Void)
|
||||
/// Wait for actions to finish
|
||||
/// - Returns: execution result of all appended actions
|
||||
func waitForResult() -> WorkerResult
|
||||
}
|
||||
|
||||
/// Worker that executes actions in pararell using DispatchGroup
|
||||
/// Warning! This implementation is not thread safe: all functions have to be called from the same thread
|
||||
class DispatchGroupParallelizationWorker: Worker {
|
||||
private let group: DispatchGroup
|
||||
private let queue: DispatchQueue
|
||||
private let qos: DispatchQoS.QoSClass
|
||||
private var observedErrors: [Error]
|
||||
|
||||
/// Default initializer
|
||||
/// - Parameter qos: QoS of the background queue to execute actions
|
||||
init(qos: DispatchQoS.QoSClass = .userInteractive) {
|
||||
group = DispatchGroup()
|
||||
queue = DispatchQueue(
|
||||
label: "DispatchGroupParallelization",
|
||||
qos: .userInteractive,
|
||||
attributes: .concurrent,
|
||||
autoreleaseFrequency: .inherit,
|
||||
target: .global(qos: qos)
|
||||
)
|
||||
observedErrors = []
|
||||
self.qos = qos
|
||||
}
|
||||
|
||||
|
||||
func appendAction(_ action: @escaping () throws -> Void) {
|
||||
group.enter()
|
||||
queue.async {
|
||||
do {
|
||||
try action()
|
||||
} catch {
|
||||
// Errors are not expected to be frequent so just enqueing another block to the working group
|
||||
self.group.enter()
|
||||
self.queue.async(group: self.group, qos: self.qos.dispatchQoS, flags: .barrier) {
|
||||
self.observedErrors.append(error)
|
||||
self.group.leave()
|
||||
}
|
||||
}
|
||||
self.group.leave()
|
||||
}
|
||||
}
|
||||
|
||||
func waitForResult() -> WorkerResult {
|
||||
group.wait()
|
||||
if observedErrors.isEmpty {
|
||||
return .successes
|
||||
}
|
||||
defer {
|
||||
observedErrors = []
|
||||
}
|
||||
return .errors(observedErrors)
|
||||
}
|
||||
}
|
||||
|
||||
extension DispatchQoS.QoSClass {
|
||||
/// Trivial transform from DispatchQoS.QoSClass to DispatchQoS
|
||||
var dispatchQoS: DispatchQoS {
|
||||
switch self {
|
||||
case .background: return .background
|
||||
case .default: return .default
|
||||
case .unspecified: return .unspecified
|
||||
case .userInitiated: return .userInitiated
|
||||
case .userInteractive: return .userInteractive
|
||||
case .utility: return .utility
|
||||
@unknown default:
|
||||
return .default
|
||||
}
|
||||
}
|
||||
}
|
||||
-64
@@ -1,64 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
// Recognizes
|
||||
protocol SwiftProductsArchitecturesRecognizer {
|
||||
/// Scans Product dir to find which final archs Xcode generated for a target
|
||||
/// Sample architecture list: ["x86_64", "x86_64-apple-ios-simulator"]
|
||||
/// - Parameters:
|
||||
/// - builtProductsDir: Location of the bulilt products dir to inspect - $(BUILT_PRODUCTS_DIR)
|
||||
/// - moduleName: a name of the module to inspect
|
||||
/// - Returns: list of architectures
|
||||
func recognizeArchitectures(builtProductsDir: URL, moduleName: String) throws -> [String]
|
||||
}
|
||||
|
||||
class DefaultSwiftProductsArchitecturesRecognizer: SwiftProductsArchitecturesRecognizer {
|
||||
/// Extension of a directory that contains all swift{module|doc|...} files
|
||||
private static let SwiftmoduleDirExtension = "swiftmodule"
|
||||
private let dirAccessor: DirAccessor
|
||||
|
||||
init(dirAccessor: DirAccessor) {
|
||||
self.dirAccessor = dirAccessor
|
||||
}
|
||||
|
||||
func recognizeArchitectures(builtProductsDir: URL, moduleName: String) throws -> [String] {
|
||||
/// Location where Xcode puts all swiftmodules
|
||||
let moduleDirectory = builtProductsDir
|
||||
.appendingPathComponent(moduleName)
|
||||
.appendingPathExtension(Self.SwiftmoduleDirExtension)
|
||||
// Skip folders (e.g. 'Project' dir that stores .sourceinfo, introduced in Xcode13)
|
||||
let productFiles = try dirAccessor.items(at: moduleDirectory).filter { url in
|
||||
try dirAccessor.itemType(atPath: url.path) == .file
|
||||
}
|
||||
/// files in a moduleDirectory have basename corresponding to the
|
||||
/// architecture (e.g. 'x86_64-apple-ios-simulator.swiftmodule', 'x86_64.swiftmodule' ...)
|
||||
let architectures = productFiles.map { file -> String in
|
||||
// recursively delete extensions to get rid of potential fingerprint overrides in a product directory
|
||||
var basenameFile = file
|
||||
while !basenameFile.pathExtension.isEmpty {
|
||||
basenameFile.deletePathExtension()
|
||||
}
|
||||
return basenameFile.lastPathComponent
|
||||
}
|
||||
// remove duplicates coming from files with different extensions (swiftmodule, swiftdoc etc.)
|
||||
return Set(architectures).sorted()
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Provider of all swift products location, expected by the Xcode
|
||||
protocol SwiftProductsLocationProvider {
|
||||
/// Destination of the ObjC header
|
||||
/// - Parameters:
|
||||
/// - targetName: target name of the swift target
|
||||
/// - moduleName: name of the module
|
||||
func objcHeaderLocation(targetName: String, moduleName: String) -> URL
|
||||
/// Destination of the .swiftmodule file
|
||||
/// - Parameters:
|
||||
/// - moduleName: name of the module
|
||||
/// - architecture: architecture of the swiftmodule
|
||||
func swiftmoduleFileLocation(moduleName: String, architecture: String) -> URL
|
||||
}
|
||||
|
||||
class DefaultSwiftProductsLocationProvider: SwiftProductsLocationProvider {
|
||||
|
||||
private let builtProductsDir: URL
|
||||
private let derivedSourcesDir: URL
|
||||
|
||||
/// Default initializer
|
||||
/// - Parameters:
|
||||
/// - builtProductsDir: current $(BUILD_PRODUCTS_DIR)
|
||||
/// - derivedSourcesDir: current $(DERIVED_SOURCES_DIR)
|
||||
init(
|
||||
builtProductsDir: URL,
|
||||
derivedSourcesDir: URL
|
||||
) {
|
||||
self.derivedSourcesDir = derivedSourcesDir
|
||||
self.builtProductsDir = builtProductsDir
|
||||
}
|
||||
|
||||
func objcHeaderLocation(targetName: String, moduleName: String) -> URL {
|
||||
// By default, Xcode generates ObjC headers for a Swift module in
|
||||
// $(DERIVED_SOURCES_DIR)/$(SWIFT_OBJC_INTERFACE_HEADER_NAME), where $(SWIFT_OBJC_INTERFACE_HEADER_NAME)
|
||||
// has a format of "\(moduleName)-Swift.h"
|
||||
// To generate a header location for some other target,
|
||||
// we need to replaced the last component of $DERIVED_SOURCES_DIR with {{targetName}}.build
|
||||
|
||||
let derivedPathDirFormat = derivedSourcesDir.lastPathComponent
|
||||
let targetSpecificDerivedSourcesDir = derivedSourcesDir
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.appendingPathComponent("\(targetName).build")
|
||||
.appendingPathComponent(derivedPathDirFormat)
|
||||
return targetSpecificDerivedSourcesDir.appendingPathComponent("\(moduleName)-Swift.h")
|
||||
}
|
||||
|
||||
func swiftmoduleFileLocation(moduleName: String, architecture: String) -> URL {
|
||||
// swiftmodule should be generated in a DerivedData's Product dir with a format:
|
||||
// "{{ModuleName}}.swiftmodule/{{arch}}.swiftmodule"
|
||||
builtProductsDir
|
||||
.appendingPathComponent("\(moduleName).swiftmodule")
|
||||
.appendingPathComponent(architecture)
|
||||
.appendingPathExtension("swiftmodule")
|
||||
}
|
||||
}
|
||||
-150
@@ -1,150 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum ThinningConsumerPostbuildPluginError: Error {
|
||||
/// The aggregation target meta misses a filekey for targets
|
||||
case missingArtifactKey(targetNames: [String])
|
||||
/// The unzipped artifact is malformed. Is misses a binary file in a root directory
|
||||
case missingBinaryForArtifact(artifact: URL)
|
||||
/// Postbuild of some target(s) failed (potentially the unzipped artifacts is broken)
|
||||
case failed(underlyingErrors: [Error])
|
||||
}
|
||||
|
||||
/// Plugin that performs "postbuild" action for all thinned targets - moves binaries, swift products, decorates with
|
||||
/// fingerprint overrides etc
|
||||
class ThinningConsumerPostbuildPlugin: ThinningConsumerPlugin, ArtifactConsumerPostbuildPlugin {
|
||||
|
||||
private let targetTempDirsRoot: URL
|
||||
private let builtProductsDir: URL
|
||||
private let productModuleName: String
|
||||
private let arch: String
|
||||
private let thinnedTargets: [String]
|
||||
private let artifactOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory
|
||||
private let swiftProductOrganizerFactory: ThinningConsumerSwiftProductsOrganizerFactory
|
||||
private let diskCopier: DiskCopier
|
||||
private let artifactInspector: ArtifactInspector
|
||||
private let swiftProductsArchitecturesRecognizer: SwiftProductsArchitecturesRecognizer
|
||||
private let worker: Worker
|
||||
|
||||
init(
|
||||
targetTempDir: URL,
|
||||
builtProductsDir: URL,
|
||||
productModuleName: String,
|
||||
arch: String,
|
||||
thinnedTargets: [String],
|
||||
artifactOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory,
|
||||
swiftProductOrganizerFactory: ThinningConsumerSwiftProductsOrganizerFactory,
|
||||
artifactInspector: ArtifactInspector,
|
||||
swiftProductsArchitecturesRecognizer: SwiftProductsArchitecturesRecognizer,
|
||||
diskCopier: DiskCopier,
|
||||
worker: Worker
|
||||
) {
|
||||
targetTempDirsRoot = targetTempDir.deletingLastPathComponent()
|
||||
self.builtProductsDir = builtProductsDir
|
||||
self.productModuleName = productModuleName
|
||||
self.arch = arch
|
||||
self.thinnedTargets = thinnedTargets
|
||||
self.artifactOrganizerFactory = artifactOrganizerFactory
|
||||
self.swiftProductOrganizerFactory = swiftProductOrganizerFactory
|
||||
self.artifactInspector = artifactInspector
|
||||
self.swiftProductsArchitecturesRecognizer = swiftProductsArchitecturesRecognizer
|
||||
self.diskCopier = diskCopier
|
||||
self.worker = worker
|
||||
}
|
||||
|
||||
/// Performs the core part of the postbuild phase for a single thinned target
|
||||
/// - Parameters:
|
||||
/// - targetName: Name of the target
|
||||
/// - productArchs: all architectures that should swift products
|
||||
/// should be generated in DerivedData's 'Products' dir
|
||||
/// - fileKey: fileKey that describes the artifact
|
||||
private func performPostbuildFor(targetName: String, productArchs archs: [String], fileKey: String) throws {
|
||||
// move all downloaded in prebuild phase
|
||||
// headers+binaries+swiftmodule(s) to the corresponding `targetName` directory
|
||||
let targetTempDir = targetTempDirsRoot.appendingPathComponent("\(targetName).build")
|
||||
let artifactOrganizer = artifactOrganizerFactory.build(targetTempDir: targetTempDir)
|
||||
let artifactLocation = artifactOrganizer.getActiveArtifactLocation()
|
||||
|
||||
// Move cached binary artifacts to the product dir
|
||||
let binaryProducts = try artifactInspector.findBinaryProducts(fromArtifact: artifactLocation)
|
||||
guard !binaryProducts.isEmpty else {
|
||||
throw ThinningConsumerPostbuildPluginError.missingBinaryForArtifact(artifact: artifactLocation)
|
||||
}
|
||||
try binaryProducts.compactMap { $0 }.forEach { product in
|
||||
try diskCopier.copy(file: product, directory: builtProductsDir)
|
||||
}
|
||||
|
||||
// Move Swift module definitions
|
||||
guard
|
||||
let moduleName = try artifactInspector.recognizeModuleName(fromArtifact: artifactLocation, arch: arch)
|
||||
else {
|
||||
/// Skip targets without swiftmodules (e.g. ObjC targets)
|
||||
return
|
||||
}
|
||||
|
||||
// Swiftmodules in an artifact are cached from the "swiftc" step. Xcode along moving the swiftmodule files
|
||||
// to the builtProductsDir, duplicates the swiftmodule definition for extra archs
|
||||
// (e.g. "x86_64" -> ["x86_64, "x86_64-apple-ios-simulator"])
|
||||
for arch in archs {
|
||||
let productsOrganizer = swiftProductOrganizerFactory.build(
|
||||
architecture: arch,
|
||||
targetName: targetName,
|
||||
moduleName: moduleName,
|
||||
artifactLocation: artifactLocation
|
||||
)
|
||||
/// fileKey is equivalent of the fingerprint
|
||||
try productsOrganizer.syncProducts(fingerprint: fileKey)
|
||||
}
|
||||
}
|
||||
|
||||
func run(meta: MainArtifactMeta) throws {
|
||||
onRun()
|
||||
// iterate all thinned targetName temp dirs and perform postbuild action
|
||||
let allCachedTargetFileKeys = ThinningPlugin.extractAllProductArtifacts(meta: meta)
|
||||
let thinnedTargetFileKeys = allCachedTargetFileKeys.filter { targetName, _ in
|
||||
thinnedTargets.contains(targetName)
|
||||
}
|
||||
// Ensure all thinned targets keys are available in a meta
|
||||
// (This is a second safety-net for. The same validation is done in the prebuild phase)
|
||||
let missedThinnedTargets = Set(thinnedTargets).subtracting(Set(thinnedTargetFileKeys.keys))
|
||||
guard missedThinnedTargets.isEmpty else {
|
||||
let targetNames = Array(missedThinnedTargets)
|
||||
let rawError = ThinningConsumerPostbuildPluginError.missingArtifactKey(targetNames: targetNames)
|
||||
// Thin project requires all artifacts to be available locally - has to fail immediately
|
||||
throw PluginError.unrecoverableError(rawError)
|
||||
}
|
||||
let archs = try swiftProductsArchitecturesRecognizer.recognizeArchitectures(
|
||||
builtProductsDir: builtProductsDir,
|
||||
moduleName: productModuleName
|
||||
)
|
||||
|
||||
for (targetName, fileKey) in thinnedTargetFileKeys {
|
||||
worker.appendAction {
|
||||
try self.performPostbuildFor(targetName: targetName, productArchs: archs, fileKey: fileKey)
|
||||
}
|
||||
}
|
||||
if case .errors(let errors) = worker.waitForResult() {
|
||||
let rawError = ThinningConsumerPostbuildPluginError.failed(underlyingErrors: errors)
|
||||
// Thin project requires all artifacts to be available locally - has to fail immediately
|
||||
throw PluginError.unrecoverableError(rawError)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum ThinningConsumerPrebuildPluginError: Error {
|
||||
/// Preparing a target(s) is not possible (potentially the artifact is not available or broken)
|
||||
case failedPreparation(underlyingErrors: [Error])
|
||||
/// TEMP_TEMP_DIR env is customised, what is not supported in a thinning mode
|
||||
case detectedOverwrittenTempDir
|
||||
/// The target that should be cached was not generated on the remote side
|
||||
case missingCachedTarget(missingTargets: [String])
|
||||
}
|
||||
|
||||
/// Prebuild plugin that downloads all thinned targets artifacts and places them in the places it would be extracted
|
||||
/// in a standard (non-thinned) prebuild step
|
||||
class ThinningConsumerPrebuildPlugin: ThinningConsumerPlugin, ArtifactConsumerPrebuildPlugin {
|
||||
private let tempDir: URL
|
||||
private let targetName: String
|
||||
private let thinnedTargets: [String]
|
||||
private let artifactsOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory
|
||||
private let networkClient: RemoteNetworkClient
|
||||
private let worker: Worker
|
||||
|
||||
/// Default initializer
|
||||
/// - Parameters:
|
||||
/// - targetName: Target name of the current (aggregation) target
|
||||
/// - tempDir: $(TEMP_DIR) of the current (aggregation) target
|
||||
/// - thinnedTargets: an array of all targets that are thinned and should be downloaded and prepared
|
||||
/// - artifactsOrganizerFactory: a factory that provides an artifact organiser
|
||||
/// - networkClient: network client used for downloading artifacts
|
||||
/// - worker: a manager that schedules blocks executions (potentially in parallel)
|
||||
init(
|
||||
targetName: String,
|
||||
tempDir: URL,
|
||||
thinnedTargets: [String],
|
||||
artifactsOrganizerFactory: ThinningConsumerArtifactsOrganizerFactory,
|
||||
networkClient: RemoteNetworkClient,
|
||||
worker: Worker
|
||||
) {
|
||||
self.targetName = targetName
|
||||
self.tempDir = tempDir
|
||||
self.thinnedTargets = thinnedTargets
|
||||
self.artifactsOrganizerFactory = artifactsOrganizerFactory
|
||||
self.networkClient = networkClient
|
||||
self.worker = worker
|
||||
}
|
||||
|
||||
/// Builds a $(TARGET_TEMP_DIR) for some other target, based on a pattern that current (aggregation) target was
|
||||
/// called with
|
||||
private func buildTempDir(forProductName otherProduct: String) throws -> URL {
|
||||
guard tempDir.lastPathComponent == "\(targetName).build" else {
|
||||
throw ThinningConsumerPrebuildPluginError.detectedOverwrittenTempDir
|
||||
}
|
||||
// Replace last component, which is exclusive for a target
|
||||
return tempDir.deletingLastPathComponent().appendingPathComponent("\(otherProduct).build")
|
||||
}
|
||||
|
||||
/// Downloads and prepares an artifact for some thinned target
|
||||
private func downloadAndPrepareArtifactFor(productName: String, fileKey: String) throws {
|
||||
let targetTempDir = try buildTempDir(forProductName: productName)
|
||||
let targetSpecificOrganizer = artifactsOrganizerFactory.build(targetTempDir: targetTempDir)
|
||||
let artifactPreparationResult = try targetSpecificOrganizer.prepareArtifactLocationFor(fileKey: fileKey)
|
||||
switch artifactPreparationResult {
|
||||
case .artifactExists(let artifactDir):
|
||||
infoLog("Artifact exists locally at \(artifactDir)")
|
||||
case .preparedForArtifact(let artifactPackage):
|
||||
infoLog("Downloading artifact to \(artifactPackage)")
|
||||
try networkClient.download(.artifact(id: fileKey), to: artifactPackage)
|
||||
|
||||
let unzippedURL = try targetSpecificOrganizer.prepare(artifact: artifactPackage)
|
||||
try targetSpecificOrganizer.activate(extractedArtifact: unzippedURL)
|
||||
}
|
||||
}
|
||||
|
||||
func run(meta: MainArtifactMeta) throws {
|
||||
onRun()
|
||||
let allArtifactFileKeys = ThinningPlugin.extractAllProductArtifacts(meta: meta)
|
||||
// Verify all thinned target's fileKeys are available in the meta
|
||||
let artifactToFetchFileKeys = allArtifactFileKeys.filter { key, _ in
|
||||
thinnedTargets.contains(key)
|
||||
}
|
||||
let missingCachedTargets = Set(thinnedTargets).subtracting(allArtifactFileKeys.keys)
|
||||
guard missingCachedTargets.isEmpty else {
|
||||
let missingTargets = Array(missingCachedTargets)
|
||||
let rawError = ThinningConsumerPrebuildPluginError.missingCachedTarget(missingTargets: missingTargets)
|
||||
// Thin project requires all artifacts to be available locally - has to fail immediately
|
||||
throw PluginError.unrecoverableError(rawError)
|
||||
}
|
||||
|
||||
for (productName, fileKey) in artifactToFetchFileKeys {
|
||||
worker.appendAction {
|
||||
try self.downloadAndPrepareArtifactFor(productName: productName, fileKey: fileKey)
|
||||
}
|
||||
}
|
||||
if case .errors(let errors) = worker.waitForResult() {
|
||||
let rawError = ThinningConsumerPrebuildPluginError.failedPreparation(underlyingErrors: errors)
|
||||
// Thin project requires all artifacts to be available locally - has to fail immediately
|
||||
throw PluginError.unrecoverableError(rawError)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
|
||||
enum ThinningCreatorPluginError: Error {
|
||||
/// Consistency error: a target with enabled XCRemoteCache doesn't contain a single artifact product. Make sure
|
||||
/// the DerivedData directory is cleared before a build
|
||||
case noSingleTargetArtifactsGenerated(rootDir: URL)
|
||||
}
|
||||
|
||||
/// Plugin that includes fileKeys of all cached targets in a target meta
|
||||
/// If scans all directories in the DerivedData to find targets that recently prepared and uploaded artifacts to the
|
||||
/// remote cache storage. It is important to enabled that plugin only for a target that is build as a last step of the
|
||||
/// building process (so it can find all relevant build products in DerivedData) for each "configuration+arch" pair
|
||||
/// Warning! This plugin assumes that producer's DerivedData are always cleaned before a build
|
||||
class ThinningCreatorPlugin: ArtifactCreatorPlugin {
|
||||
private let targetTempDir: URL
|
||||
private let modeMarkerPath: String
|
||||
private let dirScanner: DirScanner
|
||||
|
||||
/// Default Initializer
|
||||
/// - Parameter targetTempDir: Location of current target-specific temp dir (TARGET_TEMP_DIR)
|
||||
/// - Parameter modeMarkerPath: path of maker file that informs if a given target can reuse remote artifacts.
|
||||
/// - Parameter dirScanner: scanner to access disk and read files and directories hierarchy
|
||||
init(targetTempDir: URL, modeMarkerPath: String, dirScanner: DirScanner) {
|
||||
self.targetTempDir = targetTempDir
|
||||
self.modeMarkerPath = modeMarkerPath
|
||||
self.dirScanner = dirScanner
|
||||
}
|
||||
|
||||
let customPathsRemapper: DependenciesRemapper? = nil
|
||||
|
||||
func extraMetaKeys(_ meta: MainArtifactMeta) throws -> [String: String] {
|
||||
// Navigate to the root targetTempDir of all build products (for a specific Configuration+architecture)
|
||||
let allTargetsTempDirRoot = targetTempDir.deletingLastPathComponent()
|
||||
|
||||
// iterate all temp directories to find generated and uploaded artifacts. We assume that the DerivedData
|
||||
// was emptied before a build so all generated .zip files correspond to a current build
|
||||
let allURLs = try dirScanner.items(at: allTargetsTempDirRoot)
|
||||
struct TargetTuple {
|
||||
let targetName: String
|
||||
let fileKey: String
|
||||
}
|
||||
let uploadedTargetArtifacts = try allURLs.compactMap { tempDir -> TargetTuple? in
|
||||
let potentialArtifacts = try findTargetPackageZip(tempDir: tempDir)
|
||||
guard !potentialArtifacts.isEmpty else {
|
||||
// there is no generated *.zip file, so given target didn't create an artifact - it could be
|
||||
// just a helper target (like the target we integrate this plugin with)
|
||||
return nil
|
||||
}
|
||||
// Find {{fileKey}} based on the .zip file basename
|
||||
guard potentialArtifacts.count == 1 else {
|
||||
throw ThinningCreatorPluginError.noSingleTargetArtifactsGenerated(
|
||||
rootDir: tempDir
|
||||
)
|
||||
}
|
||||
let fileKey = potentialArtifacts[0].deletingPathExtension().lastPathComponent
|
||||
// Taking target name from tempDir, which has a structures "*.build"
|
||||
let targetName = tempDir.deletingPathExtension().lastPathComponent
|
||||
return TargetTuple(targetName: targetName, fileKey: fileKey)
|
||||
}
|
||||
// Build a dictionary that will be appended to the meta with a format:
|
||||
// {
|
||||
// "thinning_TargetName1": "ab2331a",
|
||||
// "thinning_TargetName2": "23a2b1b"
|
||||
// }
|
||||
let extraKeysTuples = uploadedTargetArtifacts
|
||||
.map { ("\(ThinningPlugin.fileKeyPrefix)\($0.targetName)", $0.fileKey) }
|
||||
return Dictionary(uniqueKeysWithValues: extraKeysTuples)
|
||||
}
|
||||
|
||||
private func findTargetPackageZip(tempDir: URL) throws -> [URL] {
|
||||
// Producer mode:
|
||||
// All targets that uploaded their artifacts, have it placed in the
|
||||
// `$(TARGET_TEMP_DIR)/xccache/produced/{{fileKey}}.zip` location. Find all targets that have such a file
|
||||
// ProducerFast mode:
|
||||
// If a target reused already existing artifact, it still has `$(TARGET_TEMP_DIR)/rc.enabled` marker file
|
||||
// and the reused zip is placed in:
|
||||
// `$(TARGET_TEMP_DIR)/xccache/{{fileKey}}.zip` location.
|
||||
|
||||
let targetEnabledMarker = tempDir.appendingPathComponent(modeMarkerPath)
|
||||
let targetReusedArtifactRootDir = tempDir.appendingPathComponent("xccache")
|
||||
let targetGeneratedArtifactRootDir = tempDir
|
||||
.appendingPathComponent("xccache")
|
||||
.appendingPathComponent("produced")
|
||||
|
||||
let pathToDirWithZipArtifacts: URL
|
||||
// try the `.producerFast` scenario first (the artifact was not locally
|
||||
// generated but just reused from the remote cache)
|
||||
if try dirScanner.itemType(atPath: targetEnabledMarker.path) == ItemType.file {
|
||||
pathToDirWithZipArtifacts = targetReusedArtifactRootDir
|
||||
} else {
|
||||
// cover a case when a target was build locally and an artifact
|
||||
// has just been created (locally)
|
||||
guard try dirScanner.itemType(atPath: targetGeneratedArtifactRootDir.path) == ItemType.dir else {
|
||||
// given target didn't generate any artifacts (e.g. it is never cached with XCRemoteCache)
|
||||
return []
|
||||
}
|
||||
pathToDirWithZipArtifacts = targetGeneratedArtifactRootDir
|
||||
}
|
||||
|
||||
let allFilesProduced = try dirScanner.items(at: pathToDirWithZipArtifacts)
|
||||
let allArtifacts = allFilesProduced.filter { $0.pathExtension == "zip" }
|
||||
return allArtifacts
|
||||
}
|
||||
|
||||
func artifactToUpload(main: MainArtifactMeta) throws -> [Artifact] {
|
||||
return []
|
||||
}
|
||||
}
|
||||
-84
@@ -1,84 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Generator that produces all products in the DerivedData's Products locations, using provided disk copier
|
||||
class ThinningDiskSwiftcProductsGenerator: SwiftcProductsGenerator {
|
||||
private let destinationSwiftmodulePaths: [SwiftmoduleFileExtension: URL]
|
||||
private let modulePathOutput: URL
|
||||
private let objcHeaderOutput: URL
|
||||
private let diskCopier: DiskCopier
|
||||
|
||||
init(
|
||||
modulePathOutput: URL,
|
||||
objcHeaderOutput: URL,
|
||||
diskCopier: DiskCopier
|
||||
) {
|
||||
self.modulePathOutput = modulePathOutput
|
||||
let modulePathBasename = modulePathOutput.deletingPathExtension()
|
||||
let modulePathDir = modulePathOutput.deletingLastPathComponent()
|
||||
let moduleName = modulePathBasename.lastPathComponent
|
||||
// all swiftmodule-related should be located next to the ".swiftmodule"
|
||||
// except of '.swiftsourceinfo', which should be placed in 'Project' dir
|
||||
destinationSwiftmodulePaths = Dictionary(
|
||||
uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions
|
||||
.map { ext, _ in
|
||||
switch ext {
|
||||
case .swiftsourceinfo:
|
||||
let dest = modulePathDir.appendingPathComponent("Project")
|
||||
.appendingPathComponent(moduleName)
|
||||
.appendingPathExtension(ext.rawValue)
|
||||
return (ext, dest)
|
||||
default:
|
||||
return (ext, modulePathBasename.appendingPathExtension(ext.rawValue))
|
||||
}
|
||||
}
|
||||
)
|
||||
self.objcHeaderOutput = objcHeaderOutput
|
||||
self.diskCopier = diskCopier
|
||||
}
|
||||
|
||||
func generateFrom(
|
||||
artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
|
||||
artifactSwiftModuleObjCFile: URL
|
||||
) throws -> URL {
|
||||
// Move cached -Swift.h file to the expected location
|
||||
try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput)
|
||||
for (ext, url) in sourceAtifactSwiftModuleFiles {
|
||||
let dest = destinationSwiftmodulePaths[ext]
|
||||
guard let destination = dest else {
|
||||
throw DiskSwiftcProductsGeneratorError.unknownSwiftmoduleFile
|
||||
}
|
||||
do {
|
||||
// Move cached .swiftmodule to the expected location
|
||||
try diskCopier.copy(file: url, destination: destination)
|
||||
} catch {
|
||||
if case .required = SwiftmoduleFileExtension.SwiftmoduleExtensions[ext] {
|
||||
throw error
|
||||
} else {
|
||||
infoLog("Optional .\(ext) file not found in the artifact at: \(destination.path)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build parent dir of the .swiftmodule file that contains a module
|
||||
return modulePathOutput.deletingLastPathComponent()
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Shared logic between thinning plugin producers and consumers
|
||||
enum ThinningPlugin {
|
||||
/// Prefix of the meta keys that correspond to the Thinning Plugin
|
||||
static let fileKeyPrefix = "thinning_"
|
||||
|
||||
/// Finds all artifact fileKeys from the thinned artifact meta
|
||||
/// Returns a dictionary with Product names keys and aritfact fileKey values
|
||||
static func extractAllProductArtifacts(meta: MainArtifactMeta) -> [String: String] {
|
||||
let rawKeys = meta.pluginsKeys
|
||||
|
||||
let filteredArtifacts = rawKeys.compactMap { key, value -> (String, String)? in
|
||||
guard key.hasPrefix(fileKeyPrefix) else {
|
||||
return nil
|
||||
}
|
||||
return (String(key.dropFirst(fileKeyPrefix.count)), value)
|
||||
}
|
||||
return Dictionary(uniqueKeysWithValues: filteredArtifacts)
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Abstract class for consumer's consumer and producer plugins
|
||||
class ThinningConsumerPlugin {
|
||||
private var wasRun: Bool = false
|
||||
|
||||
deinit {
|
||||
// initialised but never run plugin suggests that standard target fallbacks to the local development
|
||||
// and DerivedData still misses build artifacts.
|
||||
guard wasRun else {
|
||||
let errorMessage = """
|
||||
\(type(of: self)) plugin has never been run, thinning cannot be supported. Verify you \
|
||||
have active network connection to the remote cache server or fallback to the non-thinned mode.
|
||||
"""
|
||||
exit(1, errorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
/// called when plugin is run
|
||||
func onRun() {
|
||||
wasRun = true
|
||||
}
|
||||
}
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Moves all swift products files from the artifact to the products dir and generate fingerprint overrides
|
||||
/// In a standard flow, moving all files is done automatically by Xcode, but for a thinning flow,
|
||||
/// we need to put all .swiftmodule, .swiftdoc to the desired location in DerivedData's Products location manually
|
||||
protocol SwiftProductsOrganizer {
|
||||
func syncProducts(fingerprint: String) throws
|
||||
}
|
||||
|
||||
/// Swift products organizer that generates swift products from an unzipped artifact
|
||||
class UnzippedArtifactSwiftProductsOrganizer: SwiftProductsOrganizer {
|
||||
private let arch: String
|
||||
private let moduleName: String
|
||||
private let artifactLocation: URL
|
||||
private let productsGenerator: SwiftcProductsGenerator
|
||||
private let fingerprintSyncer: FingerprintSyncer
|
||||
|
||||
/// Default initializer
|
||||
/// - Parameters:
|
||||
/// - arch: the architecture for which the the artifact was generated
|
||||
/// - moduleName: name of the module
|
||||
/// - artifactLocation: a location of the prepared(unzipped) artifact
|
||||
/// - productsGenerator: a generator that will move files to the desired location
|
||||
/// - fingerprintSyncer: a syncer to decorate swift products with a figerprint override
|
||||
init(
|
||||
arch: String,
|
||||
moduleName: String,
|
||||
artifactLocation: URL,
|
||||
productsGenerator: SwiftcProductsGenerator,
|
||||
fingerprintSyncer: FingerprintSyncer
|
||||
) {
|
||||
self.arch = arch
|
||||
self.moduleName = moduleName
|
||||
self.artifactLocation = artifactLocation
|
||||
self.productsGenerator = productsGenerator
|
||||
self.fingerprintSyncer = fingerprintSyncer
|
||||
}
|
||||
|
||||
func syncProducts(fingerprint: String) throws {
|
||||
// Zipped artifact contains *.swiftmodule file placed in "swiftmodule/{{arch}}/{{moduleName}}.swiftmodule"
|
||||
let artifactSwiftmoduleDir = artifactLocation.appendingPathComponent("swiftmodule").appendingPathComponent(arch)
|
||||
let artifactSwiftmoduleBase = artifactSwiftmoduleDir.appendingPathComponent(moduleName)
|
||||
let artifactSwiftmoduleFiles = Dictionary(
|
||||
uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions
|
||||
.map { ext, _ in
|
||||
(ext, artifactSwiftmoduleBase.appendingPathExtension(ext.rawValue))
|
||||
}
|
||||
)
|
||||
|
||||
// -Swift.h is placed in "include/{{arch}}/{{moduleName}}/{{moduleName}-Swift.h" location
|
||||
let artifactSwiftModuleObjCFile = artifactLocation
|
||||
.appendingPathComponent("include")
|
||||
.appendingPathComponent(arch)
|
||||
.appendingPathComponent(moduleName)
|
||||
.appendingPathComponent("\(moduleName)-Swift.h")
|
||||
|
||||
let generatedModuleDir = try productsGenerator.generateFrom(
|
||||
artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
|
||||
artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
|
||||
)
|
||||
|
||||
try fingerprintSyncer.decorate(sourceDir: generatedModuleDir, fingerprint: fingerprint)
|
||||
}
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum PostbuildError: Error {
|
||||
/// Called when trying to perform the postbuild even the remote cache is disabled
|
||||
case disabledCache
|
||||
}
|
||||
|
||||
/// Performs postbuilds actions:
|
||||
/// * copies fingerprint overrides from a cache or generates these if a target was built from sources
|
||||
/// * uploads an artifact to the remote server (if the producer mode is ON)
|
||||
class Postbuild {
|
||||
private let context: PostbuildContext
|
||||
private let networkClient: RemoteNetworkClient
|
||||
private let remapper: DependenciesRemapper
|
||||
private let fingerprintAccumulator: ContextAwareFingerprintAccumulator
|
||||
private let artifactsOrganizer: ArtifactOrganizer
|
||||
private let artifactCreator: ArtifactCreator
|
||||
private let fingerprintSyncer: FingerprintSyncer
|
||||
private let dependenciesReader: DependenciesReader
|
||||
private let dependencyProcessor: DependencyProcessor
|
||||
private let fingerprintOverrideManager: FingerprintOverrideManager
|
||||
private let dSYMOrganizer: DSYMOrganizer
|
||||
private let modeController: CacheModeController
|
||||
private let metaReader: MetaReader
|
||||
private let metaWriter: MetaWriter
|
||||
private let creatorPlugins: [ArtifactCreatorPlugin]
|
||||
private let consumerPlugins: [ArtifactConsumerPostbuildPlugin]
|
||||
|
||||
init(
|
||||
context: PostbuildContext,
|
||||
networkClient: RemoteNetworkClient,
|
||||
remapper: DependenciesRemapper,
|
||||
fingerprintAccumulator: ContextAwareFingerprintAccumulator,
|
||||
artifactsOrganizer: ArtifactOrganizer,
|
||||
artifactCreator: ArtifactCreator,
|
||||
fingerprintSyncer: FingerprintSyncer,
|
||||
dependenciesReader: DependenciesReader,
|
||||
dependencyProcessor: DependencyProcessor,
|
||||
fingerprintOverrideManager: FingerprintOverrideManager,
|
||||
dSYMOrganizer: DSYMOrganizer,
|
||||
modeController: CacheModeController,
|
||||
metaReader: MetaReader,
|
||||
metaWriter: MetaWriter,
|
||||
creatorPlugins: [ArtifactCreatorPlugin],
|
||||
consumerPlugins: [ArtifactConsumerPostbuildPlugin]
|
||||
) {
|
||||
self.context = context
|
||||
self.networkClient = networkClient
|
||||
self.remapper = remapper
|
||||
self.fingerprintAccumulator = fingerprintAccumulator
|
||||
self.artifactsOrganizer = artifactsOrganizer
|
||||
self.artifactCreator = artifactCreator
|
||||
self.fingerprintSyncer = fingerprintSyncer
|
||||
self.dependenciesReader = dependenciesReader
|
||||
self.dependencyProcessor = dependencyProcessor
|
||||
self.fingerprintOverrideManager = fingerprintOverrideManager
|
||||
self.dSYMOrganizer = dSYMOrganizer
|
||||
self.modeController = modeController
|
||||
self.metaReader = metaReader
|
||||
self.metaWriter = metaWriter
|
||||
self.creatorPlugins = creatorPlugins
|
||||
self.consumerPlugins = consumerPlugins
|
||||
}
|
||||
|
||||
private func readMeta() throws -> MainArtifactMeta {
|
||||
guard case .available(commit: let remoteCommit) = context.remoteCommit else {
|
||||
throw PostbuildError.disabledCache
|
||||
}
|
||||
// Fetch meta from remote side - it should already be in the local cache, triggered by prebuild
|
||||
let metaData = try networkClient.fetch(.meta(commit: remoteCommit))
|
||||
return try metaReader.read(data: metaData)
|
||||
}
|
||||
|
||||
/// Performs all extra actions for the consumer scenario
|
||||
/// 1. Moves all fingerprint overrides from a cache dir to the product location
|
||||
/// 2. Moves all optional dSYMs to the product location
|
||||
public func performBuildCompletion() throws {
|
||||
// artifact filekey is equivalent to context specific fingerprint of its content
|
||||
let contextSpecificFingerprint = try artifactsOrganizer.getActiveArtifactFilekey()
|
||||
try generateFingerprintOverrides(contextSpecificFingerprint: contextSpecificFingerprint)
|
||||
let localArtifactLocation = artifactsOrganizer.getActiveArtifactLocation()
|
||||
try dSYMOrganizer.syncDSYM(artifactPath: localArtifactLocation)
|
||||
|
||||
// Call consumer plugins (if any)
|
||||
guard !consumerPlugins.isEmpty else {
|
||||
// quit early to not unnecessary generate meta struct
|
||||
return
|
||||
}
|
||||
let meta = try readMeta()
|
||||
try consumerPlugins.forEach { plugin in
|
||||
try plugin.run(meta: meta)
|
||||
}
|
||||
}
|
||||
|
||||
public func performBuildCleanup() throws {
|
||||
try dSYMOrganizer.cleanup()
|
||||
}
|
||||
|
||||
/// Deletes fingerprint overrides (if already set)
|
||||
public func deleteFingerprintOverrides() throws {
|
||||
try generateFingerprintOverrides(contextSpecificFingerprint: nil)
|
||||
}
|
||||
|
||||
/// Generates fingerprint overrides in the target product location, based on all files used in the compilation
|
||||
public func generateFingerprintOverrides() throws {
|
||||
// Compute a local fingerprint and decorate the .swiftmodule files
|
||||
let dependencies = try generateDependencies()
|
||||
let fingerprint = try generateFingerprint(dependencies)
|
||||
try generateFingerprintOverrides(contextSpecificFingerprint: fingerprint.contextSpecific)
|
||||
}
|
||||
|
||||
/// Uploads only a meta to the remote server - useful when the file artifact (.zip) already exists on a remote
|
||||
/// server and only a meta for a current commit sha has to be uploaded
|
||||
public func performMetaUpload(meta: MainArtifactMeta, for commit: String) throws {
|
||||
// Reset plugins keys as these are unique to each
|
||||
var meta = meta
|
||||
meta.pluginsKeys = [:]
|
||||
meta = try creatorPlugins.reduce(meta) { prevMeta, plugin in
|
||||
var meta = prevMeta
|
||||
// add extra keys from the plugin. A plugin overrides previously defined keys in case of duplication
|
||||
meta.pluginsKeys = try meta.pluginsKeys.merging(plugin.extraMetaKeys(prevMeta), uniquingKeysWith: { $1 })
|
||||
return meta
|
||||
}
|
||||
let metaPath = try metaWriter.write(meta, locationDir: context.targetTempDir)
|
||||
try networkClient.uploadSynchronously(metaPath, as: .meta(commit: commit))
|
||||
}
|
||||
|
||||
/// Builds an artifact package and uploads it to the remote server
|
||||
public func performBuildUpload(for commit: String) throws {
|
||||
let dependencies = try generateDependencies()
|
||||
let localFingerprint = try generateFingerprint(dependencies)
|
||||
// Filekey has to be unique for the context to not mix builds Debug/Release, iphonesimulator/iphoneos etc
|
||||
let fileKey = localFingerprint.contextSpecific
|
||||
// Replace all local paths to the generic ones (e.g. $SRCROOT)
|
||||
let remappers = [remapper] + creatorPlugins.compactMap(\.customPathsRemapper)
|
||||
let remapper = DependenciesRemapperComposite(remappers)
|
||||
let abstractFingerprintFiles = try remapper.replace(localPaths: dependencies.map(\.path))
|
||||
// TODO: use `inputs` read by dependenciesReader
|
||||
var meta = MainArtifactMeta(
|
||||
dependencies: abstractFingerprintFiles,
|
||||
fileKey: fileKey,
|
||||
rawFingerprint: localFingerprint.raw,
|
||||
generationCommit: commit,
|
||||
targetName: context.targetName,
|
||||
configuration: context.configuration,
|
||||
platform: context.platform,
|
||||
xcode: context.xcodeBuildNumber,
|
||||
inputs: [],
|
||||
pluginsKeys: [:]
|
||||
)
|
||||
meta = try creatorPlugins.reduce(meta) { prevMeta, plugin in
|
||||
var meta = prevMeta
|
||||
// add extra keys from the plugin. A plugin overrides previously defined keys in case of duplication
|
||||
meta.pluginsKeys = try meta.pluginsKeys.merging(plugin.extraMetaKeys(prevMeta), uniquingKeysWith: { $1 })
|
||||
return meta
|
||||
}
|
||||
|
||||
|
||||
// If a module has been built, try to decorate it with a fingerprint override
|
||||
try generateFingerprintOverrides(contextSpecificFingerprint: localFingerprint.contextSpecific)
|
||||
// Require that dSYM is generated to include in the artifact
|
||||
_ = try dSYMOrganizer.relevantDSYMLocation()
|
||||
let mainArtifact = try artifactCreator.createArtifact(artifactKey: fileKey, meta: meta)
|
||||
|
||||
// Send artifact packages with a binary (+provided by plugins) first
|
||||
// In case of a failure, don't upload meta to not mislead a consumer that the artifact is available
|
||||
let artifactsToUpload = try creatorPlugins.reduce([mainArtifact]) { prevArtifacts, plugin in
|
||||
try prevArtifacts + plugin.artifactToUpload(main: meta)
|
||||
}
|
||||
try artifactsToUpload.forEach { artifact in
|
||||
try networkClient.uploadSynchronously(artifact.package, as: .artifact(id: artifact.id))
|
||||
}
|
||||
|
||||
try networkClient.uploadSynchronously(mainArtifact.meta, as: .meta(commit: commit))
|
||||
}
|
||||
|
||||
public func controlNextRetrigger(executableURL: URL) throws {
|
||||
// If no rc.enabled is present, we disable the Postbuild Build Phase
|
||||
guard try modeController.isEnabled() else {
|
||||
try modeController.disable()
|
||||
return
|
||||
}
|
||||
// Instruct Xcode to retrigger that phase if executable has changed so fingerprint override(s) should be updated
|
||||
// TODO: consider retriggering a phase also when any of the input files has changed
|
||||
try modeController.enable(allowedInputFiles: [], dependencies: [executableURL])
|
||||
}
|
||||
|
||||
/// Reads all relevant dependencies (e.g. Xcode-embedded dependencies are skipped)
|
||||
private func generateDependencies() throws -> [URL] {
|
||||
let dependencies = try dependenciesReader.findDependencies().map(URL.init(fileURLWithPath:))
|
||||
let processedDependencies = dependencyProcessor.process(dependencies)
|
||||
let fingerprintFiles = processedDependencies.map(fingerprintOverrideManager.getFingerprintFile)
|
||||
return fingerprintFiles.map { $0.url }
|
||||
}
|
||||
|
||||
private func generateFingerprint(_ files: [URL]) throws -> Fingerprint {
|
||||
fingerprintAccumulator.reset()
|
||||
for file in files {
|
||||
do {
|
||||
try fingerprintAccumulator.append(file)
|
||||
} catch FingerprintAccumulatorError.missingFile(let content) {
|
||||
printWarning("File at \(content.path) was not found on disc. Calculating fingerprint without it.")
|
||||
}
|
||||
}
|
||||
return try fingerprintAccumulator.generate()
|
||||
}
|
||||
|
||||
/// Generates fingerprint overrides for the current module
|
||||
private func generateFingerprintOverrides(contextSpecificFingerprint: ContextSpecificFingerprint?) throws {
|
||||
// generate fingperint override only for modules (no need for ObjC targets)
|
||||
guard let modulename = context.moduleName else {
|
||||
return
|
||||
}
|
||||
try decorateSwiftmodule(modulename, contextSpecificFingerprint)
|
||||
}
|
||||
|
||||
// Add extra fingerprint override to a generated module
|
||||
private func decorateSwiftmodule(_ modulename: String, _ contextSpecificFingerprint: ContextSpecificFingerprint?) throws {
|
||||
let moduleSwiftProductURL = context.productsDir
|
||||
.appendingPathComponent(context.modulesFolderPath)
|
||||
.appendingPathComponent("\(modulename).swiftmodule")
|
||||
if let fingerprint = contextSpecificFingerprint {
|
||||
try fingerprintSyncer.decorate(
|
||||
sourceDir: moduleSwiftProductURL,
|
||||
fingerprint: fingerprint
|
||||
)
|
||||
} else {
|
||||
try fingerprintSyncer.delete(sourceDir: moduleSwiftProductURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
|
||||
enum MachOType: String, Codable {
|
||||
case staticLib
|
||||
case dynamicLib = "mh_dylib"
|
||||
case executable = "mh_execute"
|
||||
case bundle = "mh_bundle"
|
||||
case relocatable = "mh_object"
|
||||
case unknown
|
||||
}
|
||||
|
||||
enum PostbuildContextError: Error {
|
||||
/// URL address is not a valid URL
|
||||
case invalidAddress(String)
|
||||
/// ARCHS env does not contain any architecture to build
|
||||
case missingArchitecture
|
||||
}
|
||||
|
||||
public struct PostbuildContext {
|
||||
var mode: Mode
|
||||
var targetName: String
|
||||
var targetTempDir: URL
|
||||
/// Location where all compilation outputs (.o) are placed
|
||||
var compilationTempDir: URL
|
||||
var configuration: String
|
||||
var platform: String
|
||||
var productsDir: URL
|
||||
var moduleName: String?
|
||||
/// Path to the *.swiftmodule directory (irrelevant when `module` is nil). Rrelative to `productsDir`
|
||||
var modulesFolderPath: String
|
||||
var executablePath: String
|
||||
var srcRoot: URL
|
||||
var xcodeDir: URL
|
||||
var xcodeBuildNumber: String
|
||||
/// Location of the file that specifies remote commit sha
|
||||
var remoteCommitLocation: URL
|
||||
/// Commit sha of the commit to use remote cache
|
||||
var remoteCommit: RemoteCommitInfo
|
||||
var recommendedCacheAddress: URL
|
||||
/// All cache adresses to upload cache artifacts (for a producer)
|
||||
var cacheAddresses: [URL]
|
||||
/// Root directory where all statistics are stored
|
||||
var statsLocation: URL
|
||||
/// Force using the cached artifact and never fallback to the local compilation
|
||||
var forceCached: Bool
|
||||
var machOType: MachOType
|
||||
var wasDsymGenerated: Bool
|
||||
var dSYMPath: URL
|
||||
// building architecture. Used to find all dependencies from *.d files
|
||||
// Warning: if two architectures are built (e.g. for disabled "Build Archive
|
||||
// Architecture Only"), a first architecture one is picked
|
||||
let arch: String
|
||||
let builtProductsDir: URL
|
||||
/// Location to the product bundle. Can be nil for libraries
|
||||
let bundleDir: URL?
|
||||
let derivedSourcesDir: URL
|
||||
/// List of all targets to downloaded from the thinning aggregation target
|
||||
var thinnedTargets: [String]
|
||||
/// Action type: build, indexbuild etc.
|
||||
var action: BuildActionType
|
||||
let modeMarkerPath: String
|
||||
/// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
|
||||
let overlayHeadersPath: URL
|
||||
}
|
||||
|
||||
extension PostbuildContext {
|
||||
init(_ config: XCRemoteCacheConfig, env: [String: String]) throws {
|
||||
mode = config.mode
|
||||
let targetNameValue: String = try env.readEnv(key: "TARGET_NAME")
|
||||
targetName = targetNameValue
|
||||
targetTempDir = try env.readEnv(key: "TARGET_TEMP_DIR")
|
||||
let archs: [String] = try env.readEnv(key: "ARCHS").split(separator: " ").map(String.init)
|
||||
guard let firstArch = archs.first, !firstArch.isEmpty else {
|
||||
throw PostbuildContextError.missingArchitecture
|
||||
}
|
||||
arch = firstArch
|
||||
let variant: String = try env.readEnv(key: "CURRENT_VARIANT")
|
||||
compilationTempDir = try env.readEnv(key: "OBJECT_FILE_DIR_\(variant)").appendingPathComponent(arch)
|
||||
configuration = try env.readEnv(key: "CONFIGURATION")
|
||||
platform = try env.readEnv(key: "PLATFORM_NAME")
|
||||
xcodeBuildNumber = try env.readEnv(key: "XCODE_PRODUCT_BUILD_VERSION")
|
||||
productsDir = try env.readEnv(key: "TARGET_BUILD_DIR")
|
||||
moduleName = env.readEnv(key: "PRODUCT_MODULE_NAME")
|
||||
modulesFolderPath = env.readEnv(key: "MODULES_FOLDER_PATH") ?? ""
|
||||
executablePath = try env.readEnv(key: "EXECUTABLE_PATH")
|
||||
srcRoot = try env.readEnv(key: "SRCROOT")
|
||||
xcodeDir = try env.readEnv(key: "DEVELOPER_DIR")
|
||||
remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot)
|
||||
remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim())
|
||||
guard let address = URL(string: config.recommendedCacheAddress) else {
|
||||
throw PostbuildContextError.invalidAddress(config.recommendedCacheAddress)
|
||||
}
|
||||
recommendedCacheAddress = address
|
||||
statsLocation = URL(fileURLWithPath: config.statsDir.expandingTildeInPath, relativeTo: srcRoot)
|
||||
cacheAddresses = try config.cacheAddresses.map(URL.build)
|
||||
forceCached = !config.focusedTargets.isEmpty && !config.focusedTargets.contains(targetNameValue)
|
||||
machOType = try MachOType(rawValue: env.readEnv(key: "MACH_O_TYPE")) ?? .unknown
|
||||
wasDsymGenerated = try env.readEnv(key: "DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT")
|
||||
dSYMPath = try env.readEnv(key: "DWARF_DSYM_FOLDER_PATH")
|
||||
.appendingPathComponent(env.readEnv(key: "DWARF_DSYM_FILE_NAME"))
|
||||
builtProductsDir = try env.readEnv(key: "BUILT_PRODUCTS_DIR")
|
||||
if let contentsFolderPath: String = env.readEnv(key: "CONTENTS_FOLDER_PATH") {
|
||||
bundleDir = productsDir.appendingPathComponent(contentsFolderPath)
|
||||
} else {
|
||||
bundleDir = nil
|
||||
}
|
||||
derivedSourcesDir = try env.readEnv(key: "DERIVED_SOURCES_DIR")
|
||||
let thinFocusedTargetsString: String = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS") ?? ""
|
||||
thinnedTargets = thinFocusedTargetsString.split(separator: ",").map(String.init)
|
||||
action = (try? BuildActionType(rawValue: env.readEnv(key: "ACTION"))) ?? .unknown
|
||||
modeMarkerPath = config.modeMarkerPath
|
||||
/// Note: The file has yaml extension, even it is in the json format
|
||||
overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
|
||||
}
|
||||
}
|
||||
@@ -1,321 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Checks current mode from a configuration and based on that:
|
||||
/// * triggers build completion
|
||||
/// * triggers uploading artifacts to the server for a 'producer' mode
|
||||
public class XCPostbuild {
|
||||
public init() {}
|
||||
|
||||
// swiftlint:disable:next function_body_length cyclomatic_complexity
|
||||
public func main() {
|
||||
let env = ProcessInfo.processInfo.environment
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
let context: PostbuildContext
|
||||
let cacheHitLogger: CacheHitLogger
|
||||
do {
|
||||
config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
|
||||
context = try PostbuildContext(config, env: env)
|
||||
updateProcessTag(context.targetName)
|
||||
let counterFactory: FileStatsCoordinator.CountersFactory = { file, count in
|
||||
ExclusiveFileCounter(ExclusiveFile(file, mode: .override), countersCount: count)
|
||||
}
|
||||
let statsLogger = try FileStatsLogger(
|
||||
statsLocation: context.statsLocation,
|
||||
counterFactory: counterFactory,
|
||||
fileManager: fileManager
|
||||
)
|
||||
cacheHitLogger = ActionSpecificCacheHitLogger(action: context.action, statsLogger: statsLogger)
|
||||
} catch {
|
||||
exit(1, "FATAL: Postbuild initialization failed with error: \(error)")
|
||||
}
|
||||
|
||||
// Postbuild cannot disable marker, so NoopMarkerWriter used instead of a real file writer
|
||||
let modeController = PhaseCacheModeController(
|
||||
tempDir: context.targetTempDir,
|
||||
mergeCommitFile: context.remoteCommitLocation,
|
||||
phaseDependencyPath: config.postbuildDiscoveryPath,
|
||||
markerPath: config.modeMarkerPath,
|
||||
forceCached: context.forceCached,
|
||||
dependenciesWriter: FileDependenciesWriter.init,
|
||||
dependenciesReader: FileDependenciesReader.init,
|
||||
markerWriter: NoopMarkerWriter.init,
|
||||
fileManager: fileManager
|
||||
)
|
||||
|
||||
|
||||
do {
|
||||
// Initialize dependencies
|
||||
let primaryGitBranch = GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch)
|
||||
let gitClient = GitClientImpl(repoRoot: config.repoRoot, primary: primaryGitBranch, shell: shellGetStdout)
|
||||
let envsRemapper = try PathDependenciesRemapperFactory().build(
|
||||
orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
|
||||
envs: env,
|
||||
customMappings: config.outOfBandMappings
|
||||
)
|
||||
let envFingerprint = try EnvironmentFingerprintGenerator(
|
||||
configuration: config,
|
||||
env: env,
|
||||
generator: FingerprintAccumulatorImpl(algorithm: MD5Algorithm(), fileManager: fileManager)
|
||||
).generateFingerprint()
|
||||
let fingerprintFilesGenerator = FingerprintAccumulatorImpl(
|
||||
algorithm: MD5Algorithm(),
|
||||
fileManager: fileManager
|
||||
)
|
||||
let fingerprintGenerator = FingerprintGenerator(
|
||||
envFingerprint: envFingerprint,
|
||||
fingerprintFilesGenerator,
|
||||
algorithm: MD5Algorithm()
|
||||
)
|
||||
let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
|
||||
let metaWriter = JsonMetaWriter(fileWriter: fileManager, pretty: config.prettifyMetaFiles)
|
||||
let artifactCreator = BuildArtifactCreator(
|
||||
buildDir: context.productsDir,
|
||||
tempDir: context.targetTempDir,
|
||||
executablePath: context.executablePath,
|
||||
moduleName: context.moduleName,
|
||||
modulesFolderPath: context.modulesFolderPath,
|
||||
dSYMPath: context.dSYMPath,
|
||||
metaWriter: metaWriter,
|
||||
fileManager: fileManager
|
||||
)
|
||||
let dirAccessor = DirAccessorComposer(
|
||||
fileAccessor: LazyFileAccessor(fileAccessor: fileManager),
|
||||
dirScanner: fileManager
|
||||
)
|
||||
let fingerprintSyncer = FileFingerprintSyncer(
|
||||
fingerprintOverrideExtension: config.fingerprintOverrideExtension,
|
||||
dirAccessor: dirAccessor,
|
||||
extensions: config.productFilesExtensionsWithContentOverride
|
||||
)
|
||||
let sessionFactory = DefaultURLSessionFactory(config: config)
|
||||
var awsV4Signature: AWSV4Signature?
|
||||
if !config.AWSAccessKey.isEmpty {
|
||||
awsV4Signature = AWSV4Signature(
|
||||
secretKey: config.AWSSecretKey,
|
||||
accessKey: config.AWSAccessKey,
|
||||
region: config.AWSRegion,
|
||||
service: config.AWSService,
|
||||
date: Date(timeIntervalSinceNow: 0)
|
||||
)
|
||||
}
|
||||
let networkClient = NetworkClientImpl(
|
||||
session: sessionFactory.build(),
|
||||
retries: config.uploadRetries,
|
||||
fileManager: fileManager,
|
||||
awsV4Signature: awsV4Signature
|
||||
)
|
||||
let remoteNetworkClient = try RemoteNetworkClientAbstractFactory(
|
||||
mode: context.mode,
|
||||
downloadStreamURL: context.recommendedCacheAddress,
|
||||
upstreamStreamURL: context.cacheAddresses,
|
||||
networkClient: networkClient,
|
||||
urlBuilderFactory: {
|
||||
try URLBuilderImpl(
|
||||
address: $0,
|
||||
env: env,
|
||||
envFingerprint: envFingerprint,
|
||||
schemaVersion: config.schemaVersion
|
||||
)
|
||||
}
|
||||
).build()
|
||||
let fileReaderFactory: (URL) -> DependenciesReader = {
|
||||
FileDependenciesReader($0, accessor: fileManager)
|
||||
}
|
||||
let dependenciesReader = TargetDependenciesReader(
|
||||
context.compilationTempDir,
|
||||
fileDependeciesReaderFactory: fileReaderFactory,
|
||||
dirScanner: fileManager
|
||||
)
|
||||
var remappers: [DependenciesRemapper] = []
|
||||
if !config.disableVFSOverlay {
|
||||
// As the PostbuildContext assumes file location and filename (`all-product-headers.yaml`)
|
||||
// do not fail in case of a missing headers overlay file. In the future, all overlay files could be
|
||||
// captured from the swiftc invocation similarly is stored in the `history.compile` for the consumer mode.
|
||||
let overlayReader = JsonOverlayReader(
|
||||
context.overlayHeadersPath,
|
||||
mode: .bestEffort,
|
||||
fileReader: fileManager
|
||||
)
|
||||
let overlayRemapper = OverlayDependenciesRemapper(
|
||||
overlayReader: overlayReader
|
||||
)
|
||||
remappers.append(overlayRemapper)
|
||||
}
|
||||
remappers.append(envsRemapper)
|
||||
let pathRemapper = DependenciesRemapperComposite(remappers)
|
||||
let dependencyProcessor = DependencyProcessorImpl(
|
||||
xcode: context.xcodeDir,
|
||||
product: context.productsDir,
|
||||
source: context.srcRoot,
|
||||
intermediate: context.targetTempDir,
|
||||
bundle: context.bundleDir
|
||||
)
|
||||
// Override fingerprints for all produced '.swiftmodule' files
|
||||
let fingerprintOverrideManager = FingerprintOverrideManagerImpl(
|
||||
overridingFileExtensions: config.productFilesExtensionsWithContentOverride,
|
||||
fingerprintOverrideExtension: config.fingerprintOverrideExtension,
|
||||
fileManager: fileManager
|
||||
)
|
||||
|
||||
let binaryURL = context.productsDir.appendingPathComponent(context.executablePath)
|
||||
let dSYMOrganizer = DynamicDSYMOrganizer(
|
||||
productURL: binaryURL,
|
||||
machOType: context.machOType,
|
||||
dSYMPath: context.dSYMPath,
|
||||
wasDsymGenerated: context.wasDsymGenerated,
|
||||
fileManager: fileManager,
|
||||
shellCall: shellCall
|
||||
)
|
||||
let metaReader = JsonMetaReader(fileAccessor: fileManager)
|
||||
|
||||
var creatorPlugins: [ArtifactCreatorPlugin] = []
|
||||
var consumerPlugins: [ArtifactConsumerPostbuildPlugin] = []
|
||||
if config.thinningEnabled {
|
||||
// Engage all thinning plugins
|
||||
if context.moduleName == config.thinningTargetModuleName {
|
||||
switch context.mode {
|
||||
case .consumer:
|
||||
let artifactOrganizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(
|
||||
fileManager: fileManager
|
||||
)
|
||||
let swiftProductsLocationProvider =
|
||||
DefaultSwiftProductsLocationProvider(
|
||||
builtProductsDir: context.builtProductsDir,
|
||||
derivedSourcesDir: context.derivedSourcesDir
|
||||
)
|
||||
let swiftOrganizerFactory = ThinningConsumerUnzippedArtifactSwiftProductsOrganizerFactory(
|
||||
arch: context.arch,
|
||||
productsLocationProvider: swiftProductsLocationProvider,
|
||||
fingerprintSyncer: fingerprintSyncer,
|
||||
diskCopier: CopyDiskCopier(fileManager: fileManager)
|
||||
)
|
||||
let swiftProductsArchitecturesRecognizer = DefaultSwiftProductsArchitecturesRecognizer(
|
||||
dirAccessor: fileManager
|
||||
)
|
||||
let thinningPlugin = ThinningConsumerPostbuildPlugin(
|
||||
targetTempDir: context.targetTempDir,
|
||||
builtProductsDir: context.builtProductsDir,
|
||||
productModuleName: config.thinningTargetModuleName,
|
||||
arch: context.arch,
|
||||
thinnedTargets: context.thinnedTargets,
|
||||
artifactOrganizerFactory: artifactOrganizerFactory,
|
||||
swiftProductOrganizerFactory: swiftOrganizerFactory,
|
||||
artifactInspector: DefaultArtifactInspector(dirAccessor: fileManager),
|
||||
swiftProductsArchitecturesRecognizer: swiftProductsArchitecturesRecognizer,
|
||||
diskCopier: HardLinkDiskCopier(fileManager: fileManager),
|
||||
worker: DispatchGroupParallelizationWorker(qos: .userInitiated)
|
||||
)
|
||||
consumerPlugins.append(thinningPlugin)
|
||||
case .producer, .producerFast:
|
||||
let thinningPlugin = ThinningCreatorPlugin(
|
||||
targetTempDir: context.targetTempDir,
|
||||
modeMarkerPath: context.modeMarkerPath,
|
||||
dirScanner: fileManager
|
||||
)
|
||||
creatorPlugins.append(thinningPlugin)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let postbuildAction = Postbuild(
|
||||
context: context,
|
||||
networkClient: remoteNetworkClient,
|
||||
remapper: pathRemapper,
|
||||
fingerprintAccumulator: fingerprintGenerator,
|
||||
artifactsOrganizer: organizer,
|
||||
artifactCreator: artifactCreator,
|
||||
fingerprintSyncer: fingerprintSyncer,
|
||||
dependenciesReader: dependenciesReader,
|
||||
dependencyProcessor: dependencyProcessor,
|
||||
fingerprintOverrideManager: fingerprintOverrideManager,
|
||||
dSYMOrganizer: dSYMOrganizer,
|
||||
modeController: modeController,
|
||||
metaReader: metaReader,
|
||||
metaWriter: metaWriter,
|
||||
creatorPlugins: creatorPlugins,
|
||||
consumerPlugins: consumerPlugins
|
||||
)
|
||||
|
||||
// Trigger build completion
|
||||
if try modeController.isEnabled() {
|
||||
// Decorate .swiftmodule in the product dir with fingerprint(s) overrides from a cache artifact
|
||||
try postbuildAction.performBuildCompletion()
|
||||
} else if context.mode == .consumer {
|
||||
// Delete previously set overrides - they are no longer valid. The compilation was
|
||||
// done locally, most likely due to some local change
|
||||
try postbuildAction.deleteFingerprintOverrides()
|
||||
}
|
||||
|
||||
|
||||
// Trigger uploading the artifact
|
||||
switch (context.mode, try modeController.isEnabled(), context.remoteCommit) {
|
||||
case (.producerFast, true, .available(commit: let commitToReuse)):
|
||||
// Upload only updated meta. Artifact zip is already on a remote server
|
||||
let referenceCommit = try config.publishingSha ?? gitClient.getCurrentSha()
|
||||
let metaData = try remoteNetworkClient.fetch(.meta(commit: commitToReuse))
|
||||
let meta = try metaReader.read(data: metaData)
|
||||
try postbuildAction.performMetaUpload(meta: meta, for: referenceCommit)
|
||||
case (.producer, _, _), (.producerFast, _, _):
|
||||
// Generate artifacts and upload to the remote server for a reference sha
|
||||
let referenceCommit = try config.publishingSha ?? gitClient.getCurrentSha()
|
||||
try postbuildAction.performBuildUpload(for: referenceCommit)
|
||||
default:
|
||||
// Consumer does not upload anything
|
||||
break
|
||||
}
|
||||
|
||||
let executableURL = context.productsDir.appendingPathComponent(context.executablePath)
|
||||
try postbuildAction.controlNextRetrigger(executableURL: executableURL)
|
||||
|
||||
// Populate stats event for a final RC state
|
||||
// Doing it in a postmerge, as xcswiftc (and xccc) has a right to disable RC
|
||||
if try modeController.isEnabled() {
|
||||
try cacheHitLogger.logHit()
|
||||
printToUser("Cached build for \(context.targetName) target")
|
||||
} else {
|
||||
try postbuildAction.performBuildCleanup()
|
||||
try cacheHitLogger.logMiss()
|
||||
// If producers reach this point, there were no issues with publishing
|
||||
let actionName = context.mode == .consumer ? "Disabled" : "Published"
|
||||
printToUser("\(actionName) remote cache for \(context.targetName)")
|
||||
}
|
||||
} catch PluginError.unrecoverableError(let error) {
|
||||
exit(1, "\(error)")
|
||||
} catch {
|
||||
errorLog("Postbuild step failed with error: \(error)")
|
||||
if context.mode == .producer {
|
||||
// Producer cannot gracefully fail to not mark given sha as artifact-redy
|
||||
exit(1, "Postbuild step failed \(error)")
|
||||
}
|
||||
// disable postbuild until the next merge-with-primary
|
||||
do {
|
||||
try modeController.disable()
|
||||
// TODO: consider tracking errors in stats
|
||||
try cacheHitLogger.logMiss()
|
||||
printToUser("Disabled remote cache for \(context.targetName)")
|
||||
} catch {
|
||||
exit(1, "FATAL: Postbuild finishing failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum PrebuildResult: Equatable {
|
||||
case incompatible
|
||||
case compatible(localDependencies: [URL])
|
||||
}
|
||||
|
||||
/// Downloads meta info for a current commit and downloads+unzips the artifact package if fingerprints match
|
||||
class Prebuild {
|
||||
private let context: PrebuildContext
|
||||
private let networkClient: RemoteNetworkClient
|
||||
private let remapper: DependenciesRemapper
|
||||
private let fingerprintAccumulator: ContextAwareFingerprintAccumulator
|
||||
private let artifactsOrganizer: ArtifactOrganizer
|
||||
private let globalCacheSwitcher: GlobalCacheSwitcher
|
||||
private let metaReader: MetaReader
|
||||
private let artifactConsumerPrebuildPlugins: [ArtifactConsumerPrebuildPlugin]
|
||||
|
||||
init(
|
||||
context: PrebuildContext,
|
||||
networkClient: RemoteNetworkClient,
|
||||
remapper: DependenciesRemapper,
|
||||
fingerprintAccumulator: ContextAwareFingerprintAccumulator,
|
||||
artifactsOrganizer: ArtifactOrganizer,
|
||||
globalCacheSwitcher: GlobalCacheSwitcher,
|
||||
metaReader: MetaReader,
|
||||
artifactConsumerPrebuildPlugins: [ArtifactConsumerPrebuildPlugin]
|
||||
) {
|
||||
self.context = context
|
||||
self.networkClient = networkClient
|
||||
self.remapper = remapper
|
||||
self.fingerprintAccumulator = fingerprintAccumulator
|
||||
self.artifactsOrganizer = artifactsOrganizer
|
||||
self.globalCacheSwitcher = globalCacheSwitcher
|
||||
self.metaReader = metaReader
|
||||
self.artifactConsumerPrebuildPlugins = artifactConsumerPrebuildPlugins
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
public func perform() throws -> PrebuildResult {
|
||||
guard case .available(let commit) = context.remoteCommit else {
|
||||
return .incompatible
|
||||
}
|
||||
do {
|
||||
let metaData = try networkClient.fetch(.meta(commit: commit))
|
||||
let meta = try metaReader.read(data: metaData)
|
||||
let localDependencies = try remapper.replace(genericPaths: meta.dependencies).map(URL.init(fileURLWithPath:))
|
||||
let localFingerprint = try generateFingerprint(for: localDependencies)
|
||||
if localFingerprint.raw != meta.rawFingerprint {
|
||||
if context.forceCached {
|
||||
printWarning("""
|
||||
The generated target product is out-of-sync, target sources don't match the XCRemoteCache
|
||||
generated artifacts that will be used in runtime. Make sure you didn't introduce
|
||||
any modification of the target or its dependency,
|
||||
otherwise the generated application may be corrupted.
|
||||
""")
|
||||
} else {
|
||||
infoLog("""
|
||||
Local fingerprint \(localFingerprint) does not match with remote one \(meta.rawFingerprint).
|
||||
""")
|
||||
return .incompatible
|
||||
}
|
||||
}
|
||||
|
||||
let artifactPreparationResult = try artifactsOrganizer.prepareArtifactLocationFor(fileKey: meta.fileKey)
|
||||
switch artifactPreparationResult {
|
||||
case .artifactExists(let artifactDir):
|
||||
infoLog("Artifact exists locally at \(artifactDir)")
|
||||
try artifactsOrganizer.activate(extractedArtifact: artifactDir)
|
||||
case .preparedForArtifact(let artifactPackage):
|
||||
infoLog("Downloading artifact to \(artifactPackage)")
|
||||
try networkClient.download(.artifact(id: meta.fileKey), to: artifactPackage)
|
||||
|
||||
let unzippedURL = try artifactsOrganizer.prepare(artifact: artifactPackage)
|
||||
try artifactsOrganizer.activate(extractedArtifact: unzippedURL)
|
||||
infoLog("Artifact unzipped to \(unzippedURL)")
|
||||
}
|
||||
|
||||
try artifactConsumerPrebuildPlugins.forEach { plugin in
|
||||
try plugin.run(meta: meta)
|
||||
}
|
||||
return .compatible(localDependencies: localDependencies)
|
||||
} catch PluginError.unrecoverableError(let error) {
|
||||
exit(1, "\(error)")
|
||||
} catch NetworkClientError.timeout {
|
||||
if context.turnOffRemoteCacheOnFirstTimeout {
|
||||
infoLog("Network timeout observed. Falling back to local builds for all targets.")
|
||||
try globalCacheSwitcher.disable()
|
||||
}
|
||||
throw NetworkClientError.timeout
|
||||
}
|
||||
}
|
||||
|
||||
public func generateFingerprint(for files: [URL]) throws -> Fingerprint {
|
||||
try files.forEach(fingerprintAccumulator.append)
|
||||
return try fingerprintAccumulator.generate()
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum PrebuildContextError: Error {
|
||||
case missingEnv(String)
|
||||
case invalidAddress(String)
|
||||
}
|
||||
|
||||
public struct PrebuildContext {
|
||||
let targetTempDir: URL
|
||||
let productsDir: URL
|
||||
let moduleName: String?
|
||||
/// Commit sha of the commit to use remote cache
|
||||
let remoteCommit: RemoteCommitInfo
|
||||
/// Location of the file that specifies remote commit sha
|
||||
let remoteCommitLocation: URL
|
||||
let recommendedCacheAddress: URL
|
||||
/// Force using the cached artifact and never fallback to the local compilation
|
||||
let forceCached: Bool
|
||||
/// A file that stores a list of all target compilation invocations so far
|
||||
let compilationHistoryFile: URL
|
||||
/// If true, any request timeout disables remote cache for all targets
|
||||
let turnOffRemoteCacheOnFirstTimeout: Bool
|
||||
/// Name of a target
|
||||
let targetName: String
|
||||
/// List of all targets to downloaded from the thinning aggregation target
|
||||
var thinnedTargets: [String]?
|
||||
/// location of the json file that define virtual files system overlay (mappings of the virtual location file -> local file path)
|
||||
let overlayHeadersPath: URL
|
||||
}
|
||||
|
||||
extension PrebuildContext {
|
||||
init(_ config: XCRemoteCacheConfig, env: [String: String]) throws {
|
||||
targetTempDir = try env.readEnv(key: "TARGET_TEMP_DIR")
|
||||
productsDir = try env.readEnv(key: "BUILT_PRODUCTS_DIR")
|
||||
moduleName = env.readEnv(key: "PRODUCT_MODULE_NAME")
|
||||
let srcRoot: URL = try env.readEnv(key: "SRCROOT")
|
||||
remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot)
|
||||
remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim())
|
||||
guard let address = URL(string: config.recommendedCacheAddress) else {
|
||||
throw PrebuildContextError.invalidAddress(config.recommendedCacheAddress)
|
||||
}
|
||||
recommendedCacheAddress = address
|
||||
let targetName: String = try env.readEnv(key: "TARGET_NAME")
|
||||
forceCached = !config.focusedTargets.isEmpty && !config.focusedTargets.contains(targetName)
|
||||
compilationHistoryFile = targetTempDir.appendingPathComponent(config.compilationHistoryFile)
|
||||
turnOffRemoteCacheOnFirstTimeout = config.turnOffRemoteCacheOnFirstTimeout
|
||||
self.targetName = targetName
|
||||
let thinFocusedTargetsString: String? = env.readEnv(key: "SPT_XCREMOTE_CACHE_THINNED_TARGETS")
|
||||
thinnedTargets = thinFocusedTargetsString?.split(separator: ",").map(String.init)
|
||||
/// Note: The file has yaml extension, even it is in the json format
|
||||
overlayHeadersPath = targetTempDir.appendingPathComponent("all-product-headers.yaml")
|
||||
}
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class XCPrebuild {
|
||||
public init() {}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
public func main() {
|
||||
let env = ProcessInfo.processInfo.environment
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
let context: PrebuildContext
|
||||
do {
|
||||
config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
|
||||
context = try PrebuildContext(config, env: env)
|
||||
updateProcessTag(context.targetName)
|
||||
} catch {
|
||||
// Fatal error:
|
||||
exit(1, "FATAL: Prebuild initialization failed with error: \(error)")
|
||||
}
|
||||
|
||||
// Xcode may call xcprebuild phase even none of compilation files has changed (e.g. when switching between
|
||||
// simulator versions) and modifying 'mdate' of a marker file unnecessary invalidates compilation steps
|
||||
// that have to repeat their "use-from-cache" flow
|
||||
// To not introduce additional overhead, only marker writer file is saved in a lazy mode
|
||||
let lazyMarkerWriterFactory: (URL, FileManager) -> MarkerWriter = { url, fileManager in
|
||||
let lazyFileAccessor = LazyFileAccessor(fileAccessor: fileManager)
|
||||
return FileMarkerWriter(url, fileAccessor: lazyFileAccessor)
|
||||
}
|
||||
let globalCacheSwitcher = FileGlobalCacheSwitcher(context.remoteCommitLocation, fileAccessor: fileManager)
|
||||
let modeController = PhaseCacheModeController(
|
||||
tempDir: context.targetTempDir,
|
||||
mergeCommitFile: context.remoteCommitLocation,
|
||||
phaseDependencyPath: config.prebuildDiscoveryPath,
|
||||
markerPath: config.modeMarkerPath,
|
||||
forceCached: context.forceCached,
|
||||
dependenciesWriter: FileDependenciesWriter.init,
|
||||
dependenciesReader: FileDependenciesReader.init,
|
||||
markerWriter: lazyMarkerWriterFactory,
|
||||
fileManager: fileManager
|
||||
)
|
||||
|
||||
guard config.mode != .producer else {
|
||||
// Prebuild phase for a producer is noop
|
||||
// TODO: Consider a note to not adding that prebuildstep to the Xcode target
|
||||
disableRemoteCache(
|
||||
modeController: modeController,
|
||||
errorMessage: "Prebuild step disabled, selected mode: \(config.mode)"
|
||||
)
|
||||
exit(0)
|
||||
}
|
||||
|
||||
guard !modeController.shouldDisable(for: context.remoteCommit) else {
|
||||
// Previous RC runs explicitly disabled using remote cache for that remote sha
|
||||
// Short-circut early all `xc*` apps until remote commit change
|
||||
disableRemoteCache(
|
||||
modeController: modeController,
|
||||
errorMessage: "Prebuild step was disabled for current commit: \(context.remoteCommit)"
|
||||
)
|
||||
exit(0)
|
||||
}
|
||||
|
||||
do {
|
||||
let envFingerprint = try EnvironmentFingerprintGenerator(
|
||||
configuration: config,
|
||||
env: env,
|
||||
generator: FingerprintAccumulatorImpl(algorithm: MD5Algorithm(), fileManager: fileManager)
|
||||
).generateFingerprint()
|
||||
let urlBuilder = try URLBuilderImpl(
|
||||
address: context.recommendedCacheAddress,
|
||||
env: env,
|
||||
envFingerprint: envFingerprint,
|
||||
schemaVersion: config.schemaVersion
|
||||
)
|
||||
let sessionFactory = DefaultURLSessionFactory(config: config)
|
||||
var awsV4Signature: AWSV4Signature?
|
||||
if !config.AWSAccessKey.isEmpty {
|
||||
awsV4Signature = AWSV4Signature(
|
||||
secretKey: config.AWSSecretKey,
|
||||
accessKey: config.AWSAccessKey,
|
||||
region: config.AWSRegion,
|
||||
service: config.AWSService,
|
||||
date: Date(timeIntervalSinceNow: 0)
|
||||
)
|
||||
}
|
||||
let networkClient = NetworkClientImpl(
|
||||
session: sessionFactory.build(),
|
||||
retries: config.downloadRetries,
|
||||
fileManager: fileManager,
|
||||
awsV4Signature: awsV4Signature
|
||||
)
|
||||
let cacheURL: URL = fileManager.urls(for: .cachesDirectory, in: .userDomainMask).first!
|
||||
let cacheURLBuilder = LocalURLBuilderImpl(cachePath: cacheURL)
|
||||
let cacheNetworkClient = CachedNetworkClient(
|
||||
localURLBuilder: cacheURLBuilder,
|
||||
client: networkClient,
|
||||
fileManager: fileManager
|
||||
)
|
||||
let client: NetworkClient = config.disableHttpCache ? networkClient : cacheNetworkClient
|
||||
let remoteNetworkClient = RemoteNetworkClientImpl(client, urlBuilder)
|
||||
let envsRemapper = try PathDependenciesRemapperFactory().build(
|
||||
orderKeys: DependenciesMapping.rewrittenEnvs + config.customRewriteEnvs,
|
||||
envs: env,
|
||||
customMappings: config.outOfBandMappings
|
||||
)
|
||||
var remappers: [DependenciesRemapper] = []
|
||||
if !config.disableVFSOverlay {
|
||||
// As PrebuildContext assumes file location and its filename (`all-product-headers.yaml`)
|
||||
// do not fail in case of a missing headers overlay file.
|
||||
let overlayReader = JsonOverlayReader(
|
||||
context.overlayHeadersPath,
|
||||
mode: .bestEffort,
|
||||
fileReader: fileManager
|
||||
)
|
||||
let overlayRemapper = OverlayDependenciesRemapper(
|
||||
overlayReader: overlayReader
|
||||
)
|
||||
remappers.append(overlayRemapper)
|
||||
}
|
||||
remappers.append(envsRemapper)
|
||||
let pathRemapper = DependenciesRemapperComposite(remappers)
|
||||
let filesFingerprintGenerator = FingerprintAccumulatorImpl(
|
||||
algorithm: MD5Algorithm(),
|
||||
fileManager: fileManager
|
||||
)
|
||||
let fingerprintGenerator = FingerprintGenerator(
|
||||
envFingerprint: envFingerprint,
|
||||
filesFingerprintGenerator,
|
||||
algorithm: MD5Algorithm()
|
||||
)
|
||||
let organizer = ZipArtifactOrganizer(targetTempDir: context.targetTempDir, fileManager: fileManager)
|
||||
let compilationHistoryOrganizer = CompilationHistoryFileOrganizer(
|
||||
context.compilationHistoryFile,
|
||||
fileManager: fileManager
|
||||
)
|
||||
let metaReader = JsonMetaReader(fileAccessor: fileManager)
|
||||
var consumerPlugins: [ArtifactConsumerPrebuildPlugin] = []
|
||||
|
||||
if config.thinningEnabled {
|
||||
if context.moduleName == config.thinningTargetModuleName, let thinnedTarget = context.thinnedTargets {
|
||||
let organizerFactory = ThinningConsumerZipArtifactsOrganizerFactory(fileManager: .default)
|
||||
let aggregationPlugin = ThinningConsumerPrebuildPlugin(
|
||||
targetName: context.targetName,
|
||||
tempDir: context.targetTempDir,
|
||||
thinnedTargets: thinnedTarget,
|
||||
artifactsOrganizerFactory: organizerFactory,
|
||||
networkClient: remoteNetworkClient,
|
||||
worker: DispatchGroupParallelizationWorker(qos: .userInitiated)
|
||||
)
|
||||
consumerPlugins.append(aggregationPlugin)
|
||||
}
|
||||
}
|
||||
|
||||
let prebuildAction = Prebuild(
|
||||
context: context,
|
||||
networkClient: remoteNetworkClient,
|
||||
remapper: pathRemapper,
|
||||
fingerprintAccumulator: fingerprintGenerator,
|
||||
artifactsOrganizer: organizer,
|
||||
globalCacheSwitcher: globalCacheSwitcher,
|
||||
metaReader: metaReader,
|
||||
artifactConsumerPrebuildPlugins: consumerPlugins
|
||||
)
|
||||
|
||||
let actionResult = try prebuildAction.perform()
|
||||
switch actionResult {
|
||||
case .incompatible:
|
||||
infoLog("Remote cache cannot be used")
|
||||
try modeController.disable()
|
||||
case .compatible(localDependencies: let dependencies):
|
||||
// TODO: pass `allowedInputFiles` observed in the build time
|
||||
try modeController.enable(allowedInputFiles: dependencies, dependencies: dependencies)
|
||||
compilationHistoryOrganizer.reset()
|
||||
}
|
||||
} catch {
|
||||
disableRemoteCache(
|
||||
modeController: modeController,
|
||||
errorMessage: "Prebuild step failed with error: \(error)"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private func disableRemoteCache(modeController: PhaseCacheModeController, errorMessage: String?) {
|
||||
if let message = errorMessage {
|
||||
errorLog(message)
|
||||
}
|
||||
do {
|
||||
try modeController.disable()
|
||||
} catch {
|
||||
exit(1, "FATAL: Prebuild fallback to source-mode failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,521 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Builds a cc (clang) command wrapper that noops when cached artifact is used
|
||||
protocol CCWrapperBuilder {
|
||||
/// Compiles CC wrapper and places output binary in a destination location
|
||||
/// - Parameters:
|
||||
/// - destination: output location of the binary
|
||||
/// - commitSha: remote commit sha that is currently in use
|
||||
func compile(to destination: URL, commitSha: String) throws
|
||||
}
|
||||
|
||||
// swiftlint:disable:next type_body_length
|
||||
class TemplateBasedCCWrapperBuilder: CCWrapperBuilder {
|
||||
private static let AppleGenericVersioningSuffix = "_vers.c"
|
||||
private let clangCommand: String
|
||||
private let markerPath: String
|
||||
private let cachedTargetMockFilename: String
|
||||
private let prebuildDFilename: String
|
||||
private let compilationHistoryFilename: String
|
||||
private let shell: ShellOutFunction
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(
|
||||
clangCommand: String,
|
||||
markerPath: String,
|
||||
cachedTargetMockFilename: String,
|
||||
prebuildDFilename: String,
|
||||
compilationHistoryFilename: String,
|
||||
shellOut: @escaping ShellOutFunction,
|
||||
fileManager: FileManager
|
||||
) {
|
||||
self.clangCommand = clangCommand
|
||||
self.markerPath = markerPath
|
||||
self.cachedTargetMockFilename = cachedTargetMockFilename
|
||||
self.prebuildDFilename = prebuildDFilename
|
||||
self.compilationHistoryFilename = compilationHistoryFilename
|
||||
shell = shellOut
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
/// Compiles xccc app and places binary in the destination location
|
||||
func compile(to destination: URL, commitSha: String) throws {
|
||||
let compilationFile = fileManager.temporaryDirectory.appendingPathComponent("xccc.c")
|
||||
let compilationContent = buildWrapperSource(
|
||||
clangCommand: clangCommand,
|
||||
markerFilename: markerPath,
|
||||
commitSha: commitSha
|
||||
)
|
||||
fileManager.createFile(
|
||||
atPath: compilationFile.path,
|
||||
contents: compilationContent.data(using: .utf8),
|
||||
attributes: nil
|
||||
)
|
||||
infoLog("ClangWrapperBuilder compiles file at \(compilationFile).")
|
||||
// -O3: optimize for faster execution
|
||||
let args = [clangCommand, "-arch", "arm64", "-arch", "x86_64", "-O3", compilationFile.path, "-o", destination.path]
|
||||
let compilationOutput = try shell("xcrun", args, URL(fileURLWithPath: "").path, nil)
|
||||
infoLog("Clang compilation output: \(compilationOutput)")
|
||||
}
|
||||
|
||||
|
||||
/// Generates source of the cc wrapper
|
||||
// swiftlint:disable line_length
|
||||
// swiftlint:disable:next function_body_length
|
||||
private func buildWrapperSource(clangCommand: String, markerFilename: String, commitSha: String) -> String {
|
||||
return """
|
||||
/**
|
||||
Clang compiler wrapper manages compilation. When a marker file, placed in the `-MF/../../../\(markerFilename)`:
|
||||
1) is missing - fallback to \(clangCommand)
|
||||
2) exists - creates empty .o file and creates .d with the same content as a marker
|
||||
(which is expected to be in the .d format)
|
||||
3) otherwise, return 1 and prints a message to the error stream.
|
||||
*/
|
||||
|
||||
#include <fcntl.h> /* For system call open */
|
||||
#include <libgen.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/// checks if string str has suffix prefix
|
||||
int isSuffixed(const char *str, const char *suffix)
|
||||
{
|
||||
int suffix_len = strlen(suffix);
|
||||
return (strlen(str) > suffix_len && !strcmp(str + strlen(str) - suffix_len, suffix));
|
||||
}
|
||||
|
||||
void createFile(const char *path, const char *content)
|
||||
{
|
||||
FILE *fp;
|
||||
fp = fopen(path, "wb");
|
||||
if (content) {
|
||||
fwrite(content, 1, strlen(content), fp);
|
||||
}
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
void createEmptyFile(const char *path)
|
||||
{
|
||||
createFile(path, NULL);
|
||||
}
|
||||
|
||||
/// Writes empty .dia with no diagonostics messages (no errors, no warnings)
|
||||
/// Clang implementation: https://clang.llvm.org/doxygen/SerializedDiagnosticPrinter_8cpp_source.html
|
||||
void createPlaceholderDiaFile(const char *path)
|
||||
{
|
||||
// empty .dia file dumped using `xxd --include empty_sample.dia`
|
||||
unsigned char empty_dia[] = {
|
||||
0x44, 0x49, 0x41, 0x47, 0x01, 0x08, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00,
|
||||
0x07, 0x01, 0xb2, 0x40, 0xb4, 0x42, 0x39, 0xd0, 0x43, 0x38, 0x3c, 0x20,
|
||||
0x81, 0x2d, 0x94, 0x83, 0x3c, 0xcc, 0x43, 0x3a, 0xbc, 0x83, 0x3b, 0x1c,
|
||||
0x04, 0x88, 0x62, 0x80, 0x40, 0x71, 0x10, 0x24, 0x0b, 0x04, 0x29, 0xa4,
|
||||
0x43, 0x38, 0x9c, 0xc3, 0x43, 0x22, 0x90, 0x42, 0x3a, 0x84, 0xc3, 0x39,
|
||||
0xa4, 0x82, 0x3b, 0x98, 0xc3, 0x3b, 0x3c, 0x24, 0xc3, 0x2c, 0xc8, 0xc3,
|
||||
0x38, 0xc8, 0x42, 0x38, 0xb8, 0xc3, 0x39, 0x94, 0xc3, 0x03, 0x52, 0x8c,
|
||||
0x42, 0x38, 0xd0, 0x83, 0x2b, 0x84, 0x43, 0x3b, 0x94, 0xc3, 0x43, 0x42,
|
||||
0x90, 0x42, 0x3a, 0x84, 0xc3, 0x39, 0x98, 0x02, 0x3b, 0x84, 0xc3, 0x39,
|
||||
0x3c, 0x24, 0x86, 0x29, 0xa4, 0x03, 0x3b, 0x94, 0x83, 0x2b, 0x84, 0x43,
|
||||
0x3b, 0x94, 0xc3, 0x83, 0x71, 0x98, 0x42, 0x3a, 0xe0, 0x43, 0x2a, 0xd0,
|
||||
0xc3, 0x41, 0x90, 0xa8, 0x0a, 0xc8, 0x10, 0x25, 0x50, 0x08, 0x14, 0x02,
|
||||
0x85, 0x28, 0x51, 0x04, 0x83, 0x4a, 0x16, 0x08, 0x0c, 0x82, 0xd4, 0x74,
|
||||
0x40, 0x94, 0x40, 0x21, 0x50, 0x08, 0x14, 0xa2, 0x04, 0x0a, 0x81, 0x42,
|
||||
0xa0, 0x90, 0x24, 0x10, 0x25, 0x30, 0xa8, 0xa6, 0x81, 0x28, 0x81, 0x42,
|
||||
0xa0, 0x10, 0x18, 0xd4, 0xf5, 0x40, 0x94, 0x40, 0x21, 0x50, 0x08, 0x14,
|
||||
0xa2, 0x04, 0x0a, 0x81, 0x42, 0xa0, 0x10, 0x18, 0x14, 0x00, 0x00, 0x00,
|
||||
0x21, 0x0c, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00
|
||||
};
|
||||
unsigned int empty_dia_len = 220;
|
||||
FILE *fp;
|
||||
fp = fopen(path, "wb");
|
||||
fwrite(empty_dia, 1, empty_dia_len, fp);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
bool fileExists(const char *path)
|
||||
{
|
||||
struct stat buffer;
|
||||
return (stat(path, &buffer) == 0);
|
||||
}
|
||||
|
||||
/// Copies content from sourcePath to the destination (creates one if doesn't exists)
|
||||
/// Returns true for a success, false otherwise
|
||||
bool copyFile(const char *sourcePath, const char *destinationPath)
|
||||
{
|
||||
int buf_size = 512;
|
||||
char buffer[buf_size];
|
||||
size_t size;
|
||||
FILE *source = fopen(sourcePath, "r");
|
||||
FILE *destination = fopen(destinationPath, "wb");
|
||||
|
||||
if (source == NULL || destination == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
while ((size = fread(buffer, 1, buf_size, source)) > 0) {
|
||||
fwrite(buffer, 1, size, destination);
|
||||
}
|
||||
|
||||
fclose(source);
|
||||
fclose(destination);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool isPresentInFile(const char *filePath, const char *search_line)
|
||||
{
|
||||
FILE * fp;
|
||||
char * line = NULL;
|
||||
size_t len = 0;
|
||||
ssize_t read;
|
||||
bool found = false;
|
||||
size_t search_len = strlen(search_line);
|
||||
|
||||
fp = fopen(filePath, "r");
|
||||
if (fp == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
while ((read = getline(&line, &len, fp)) != -1) {
|
||||
// Check if a line starts with search_line followed by "" (last entry) or " \\\\n" (otherwise)
|
||||
if (strncasecmp(line, search_line, search_len) == 0 &&
|
||||
(strcmp(line + search_len, "") == 0 || strcmp(line + search_len, " \\\\\\n") == 0)
|
||||
) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
free(line);
|
||||
fclose(fp);
|
||||
return found;
|
||||
}
|
||||
|
||||
/// Decides if the file is a valid compilation unit, required to be considered in the allowed input files
|
||||
/// If not, compilation step can be safelly skip for the consumer mode
|
||||
bool isIrrelevantFile(const char *filePath)
|
||||
{
|
||||
// Skip Apple Generic Versioning file "{TargetName}_vers.c", generated during the compilation step
|
||||
return isSuffixed(filePath, "\(Self.AppleGenericVersioningSuffix)");
|
||||
}
|
||||
|
||||
/// Builds a concatenation strings. First string s1 may contain NULL characters
|
||||
/// Returns the size of the output 'string'
|
||||
size_t concat(char *s1, size_t s1_len, const char *s2, char **output)
|
||||
{
|
||||
size_t concat_len = strlen(s2);
|
||||
const size_t size = s1_len + concat_len + 1;
|
||||
char *new = realloc(s1, size);
|
||||
memcpy(new + s1_len, s2, concat_len + 1);
|
||||
*output = new;
|
||||
return size - 1;
|
||||
}
|
||||
|
||||
/// Adds NULL byte to the string s1
|
||||
/// Returns the size of the output 'string'
|
||||
size_t addZero(char *s1, size_t len, char **output)
|
||||
{
|
||||
char *delimiter = "\\0";
|
||||
const size_t size = len + 1 + 1;
|
||||
char *new = realloc(s1, size);
|
||||
memcpy(new + len, delimiter, 1 + 1);
|
||||
*output = new;
|
||||
return size - 1;
|
||||
}
|
||||
|
||||
bool appendCallToFile(const char *filePath, const char * args[], int len)
|
||||
{
|
||||
int fd = open(filePath, O_WRONLY|O_APPEND);
|
||||
|
||||
if (fd == -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// prepare a string command to store
|
||||
size_t command_len = 0;
|
||||
char *command = NULL;
|
||||
|
||||
// print all arguments followed by {0x0}
|
||||
for (int i = 0 ; i < len; i++) {
|
||||
command_len = concat(command, command_len, args[i], &command);
|
||||
command_len = addZero(command, command_len, &command);
|
||||
}
|
||||
|
||||
// Finish with NULL to mirror execv format that expects NULL element at the end
|
||||
command_len = addZero(command, command_len, &command);
|
||||
// finish a command with a new line character
|
||||
command_len = concat(command, command_len, "\\n", &command);
|
||||
|
||||
// acquire a lock
|
||||
if (flock(fd, LOCK_EX) == -1) {
|
||||
close(fd);
|
||||
free(command);
|
||||
return false;
|
||||
}
|
||||
|
||||
struct stat st0;
|
||||
fstat(fd, &st0);
|
||||
if (st0.st_nlink == 0) {
|
||||
// the file has been deleted, local compilation should happen
|
||||
flock(fd, LOCK_UN);
|
||||
close(fd);
|
||||
free(command);
|
||||
return false;
|
||||
}
|
||||
write(fd, command, command_len);
|
||||
free(command);
|
||||
|
||||
if (flock(fd, LOCK_UN) == -1) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
close(fd);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Builds an array of strings from contiguous set of strings, terminated with NULL element
|
||||
/// e.g. 'a{0x0}b{0x0}{0x0}' -> ['a', 'b', NULL]
|
||||
char **buildArrayFromContiguousString(char *str) {
|
||||
char **pointer = NULL;
|
||||
char *pos = str;
|
||||
int count = 0;
|
||||
while (true) {
|
||||
size_t len = strlen(pos);
|
||||
count += 1;
|
||||
pointer = realloc(pointer, count * sizeof(char*));
|
||||
pointer[count - 1] = pos;
|
||||
pos += (len + 1);
|
||||
if (len == 0 ) {
|
||||
return pointer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calls all commands stored in filePath location
|
||||
void fallbackPreviousCalls(const char *filePath)
|
||||
{
|
||||
int fd = open(filePath, O_RDONLY);
|
||||
|
||||
if (fd == -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
// acquire a lock
|
||||
if (flock(fd, LOCK_EX) == -1) {
|
||||
close(fd);
|
||||
return;
|
||||
}
|
||||
|
||||
// make sure the file still exists, it might be deleted while we were waiting for a lock
|
||||
if (access(filePath, F_OK) == -1) {
|
||||
close(fd);
|
||||
return;
|
||||
}
|
||||
|
||||
struct stat st0;
|
||||
fstat(fd, &st0);
|
||||
if(st0.st_nlink == 0) {
|
||||
// the file has been deleted - no need to fallback anything else
|
||||
flock(fd, LOCK_UN);
|
||||
close(fd);
|
||||
return;
|
||||
}
|
||||
|
||||
char * line = NULL;
|
||||
size_t len = 0;
|
||||
|
||||
// iterate all lines in a file and execute commands one-by-one
|
||||
FILE * file = fdopen(fd, "r");
|
||||
ssize_t read;
|
||||
while ((read = getline(&line, &len, file)) != -1) {
|
||||
// Call all clang invocations one-by-one
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
// child process
|
||||
char **array = buildArrayFromContiguousString(line);
|
||||
// forked process
|
||||
execvp(line, array);
|
||||
} else {
|
||||
// hosting process
|
||||
int stat;
|
||||
wait(&stat);
|
||||
if (!WIFEXITED(stat)) {
|
||||
//the command finish incorrectly
|
||||
exit(1);
|
||||
}
|
||||
if (WEXITSTATUS(stat)) {
|
||||
// error in the "clang" call, quit with a status code
|
||||
exit(WEXITSTATUS(stat));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
free(line);
|
||||
remove(filePath);
|
||||
flock(fd, LOCK_UN);
|
||||
fclose(file);
|
||||
close(fd);
|
||||
}
|
||||
|
||||
int main(int argc, const char * argv[])
|
||||
{
|
||||
const char *dependency_arg_name = "-MF";
|
||||
const char *output_arg_name = "-o";
|
||||
const char *serialize_diagnostics_arg_name = "--serialize-diagnostics";
|
||||
const char *clang_cmd = "\(clangCommand)";
|
||||
const char *markerFile = "\(markerFilename)";
|
||||
const char *compilationHistoryFile = "\(compilationHistoryFilename)";
|
||||
const char *prebuildDFile = "\(prebuildDFilename)";
|
||||
|
||||
|
||||
// null termination args
|
||||
const char *clang_args[argc + 1];
|
||||
clang_args[0] = clang_cmd;
|
||||
|
||||
const char *dependency_file = NULL;
|
||||
const char *output_file= NULL;
|
||||
const char *input_file = NULL;
|
||||
const char *diagnostics_file = NULL;
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (strcmp(argv[i], dependency_arg_name) == 0 && i < (argc - 1) ) {
|
||||
// called with "-MF path" pattern and not the last argument
|
||||
clang_args[i] = argv[i];
|
||||
i += 1;
|
||||
clang_args[i] = argv[i];
|
||||
dependency_file = argv[i];
|
||||
} else if (strcmp(argv[i], output_arg_name) == 0 && i < (argc - 1) ) {
|
||||
// called with "-o path" pattern and not the last argument
|
||||
clang_args[i] = argv[i];
|
||||
i += 1;
|
||||
clang_args[i] = argv[i];
|
||||
output_file = argv[i];
|
||||
} if (strcmp(argv[i], serialize_diagnostics_arg_name) == 0 && i < (argc - 1) ) {
|
||||
// called with "--serialize-diagnostics path" pattern and not the last argument
|
||||
clang_args[i] = argv[i];
|
||||
i += 1;
|
||||
clang_args[i] = argv[i];
|
||||
diagnostics_file = argv[i];
|
||||
} else if (
|
||||
isSuffixed(argv[i],".m") ||
|
||||
isSuffixed(argv[i],".mm") ||
|
||||
isSuffixed(argv[i],".c") ||
|
||||
isSuffixed(argv[i],".cc") ||
|
||||
isSuffixed(argv[i],".cpp") ||
|
||||
isSuffixed(argv[i],".c++") ||
|
||||
isSuffixed(argv[i],".cxx")
|
||||
) {
|
||||
// a full list of extensions is taken from https://clang.llvm.org/docs/ClangFormatStyleOptions.html
|
||||
// support for .m,.mm,.c,.cc,.cpp,.c++,.cxx input files
|
||||
clang_args[i] = argv[i];
|
||||
input_file = argv[i];
|
||||
} else {
|
||||
// pass original parameter transparently
|
||||
clang_args[i] = argv[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Verify all input arguments
|
||||
if (dependency_file == NULL) {
|
||||
fprintf(stderr, "error: missing %s input\\n", dependency_arg_name);
|
||||
exit(1);
|
||||
}
|
||||
if (output_file == NULL) {
|
||||
fprintf(stderr, "error: missing %s input\\n", output_arg_name);
|
||||
exit(1);
|
||||
}
|
||||
if (input_file == NULL) {
|
||||
fprintf(stderr, "error: missing input file\\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
// Find tmp_dir
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers"
|
||||
const char *tmp_dir = dirname(dirname(dirname(dependency_file)));
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
// Find input in allowed files
|
||||
char marker_path[1024];
|
||||
sprintf(marker_path, "%s/%s", tmp_dir, markerFile);
|
||||
|
||||
// A file that keeps all clang invocations
|
||||
char compilation_history_path[1024];
|
||||
sprintf(compilation_history_path, "%s/%s", tmp_dir, compilationHistoryFile);
|
||||
|
||||
// Path of the prebuild.d dependency file
|
||||
char prebuild_d_path[1024];
|
||||
sprintf(prebuild_d_path, "%s/%s", tmp_dir, prebuildDFile);
|
||||
|
||||
if (fileExists(marker_path))
|
||||
{
|
||||
if (
|
||||
isIrrelevantFile(input_file) ||
|
||||
isPresentInFile(marker_path, input_file) ||
|
||||
isSuffixed(input_file, "\(cachedTargetMockFilename).m")
|
||||
) {
|
||||
// Save .d files (copy a marker file)
|
||||
bool copyResult = copyFile(marker_path, dependency_file);
|
||||
if (!copyResult) {
|
||||
fprintf(stderr, "error: .d file generation failed.\\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Create empty .o file
|
||||
createEmptyFile(output_file);
|
||||
// Create .dia file (if specified)
|
||||
if (diagnostics_file != NULL) {
|
||||
createPlaceholderDiaFile(diagnostics_file);
|
||||
}
|
||||
// add to compilation_history_path file so other clang execution can retrigger it if a new file is found
|
||||
bool appended = appendCallToFile(compilation_history_path, clang_args, argc);
|
||||
if (appended) {
|
||||
exit(0);
|
||||
}
|
||||
// Failed to save, most likely some other clang fallbacked to the local compilation already
|
||||
// so local compilation should happen
|
||||
} else {
|
||||
// disable remote cache first to trigger prebuild phase in the next build
|
||||
remove(marker_path);
|
||||
// stop trying to reuse artifact for this specific remote commit sha
|
||||
createFile(prebuild_d_path, "\(FileDependenciesWriter.skipForShaKey): \(commitSha)\\n");
|
||||
// read from compilation_history_path, execute one by one all invocations
|
||||
fallbackPreviousCalls(compilation_history_path);
|
||||
}
|
||||
}
|
||||
|
||||
// null-terminating the args array
|
||||
clang_args[argc] = NULL;
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wincompatible-pointer-types-discards-qualifiers"
|
||||
/// execvp takes $PATH to consideration
|
||||
return execvp(clang_cmd, clang_args);
|
||||
#pragma GCC diagnostic pop
|
||||
}
|
||||
"""
|
||||
} // swiftlint:disable:next file_length line_length
|
||||
} // swiftlint:enable line_length
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
typealias BuildSettings = [String: Any]
|
||||
|
||||
// Manages Xcode build settings
|
||||
protocol BuildSettingsIntegrateAppender {
|
||||
/// Appends XCRemoteCache-specific build settings
|
||||
/// - Parameters:
|
||||
/// - buildSettings: original build settings
|
||||
/// - wrappers: definition of XCRemoteCache binaries location
|
||||
func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings
|
||||
}
|
||||
|
||||
class XcodeProjBuildSettingsIntegrateAppender: BuildSettingsIntegrateAppender {
|
||||
private let mode: Mode
|
||||
private let repoRoot: URL
|
||||
|
||||
init(mode: Mode, repoRoot: URL) {
|
||||
self.mode = mode
|
||||
self.repoRoot = repoRoot
|
||||
}
|
||||
|
||||
func appendToBuildSettings(buildSettings: BuildSettings, wrappers: XCRCBinariesPaths) -> BuildSettings {
|
||||
var result = buildSettings
|
||||
result["SWIFT_EXEC"] = wrappers.swiftc.path
|
||||
// When generating artifacts, no need to shell-out all compilation commands to our wrappers
|
||||
if case .consumer = mode {
|
||||
result["CC"] = wrappers.cc.path
|
||||
result["LD"] = wrappers.ld.path
|
||||
result["LIBTOOL"] = wrappers.libtool.path
|
||||
}
|
||||
|
||||
let existingSwiftFlags = result["OTHER_SWIFT_FLAGS"] as? String
|
||||
let existingCFlags = result["OTHER_CFLAGS"] as? String
|
||||
var swiftFlags = XcodeSettingsSwiftFlags(settingValue: existingSwiftFlags)
|
||||
var clangFlags = XcodeSettingsCFlags(settingValue: existingCFlags)
|
||||
|
||||
// Overriding debug prefix map for Swift and ObjC to have consistent absolute path for all debug symbols
|
||||
swiftFlags.assignFlag(key: "debug-prefix-map", value: "\(repoRoot.path)=$(XCRC_FAKE_SRCROOT)")
|
||||
clangFlags.assignFlag(key: "debug-prefix-map", value: "\(repoRoot.path)=$(XCRC_FAKE_SRCROOT)")
|
||||
|
||||
result["OTHER_SWIFT_FLAGS"] = swiftFlags.settingValue
|
||||
result["OTHER_CFLAGS"] = clangFlags.settingValue
|
||||
|
||||
result["XCRC_FAKE_SRCROOT"] = "/\(String(repeating: "x", count: 10))"
|
||||
result["XCRC_PLATFORM_PREFERRED_ARCH"] = "$(LINK_FILE_LIST_$(CURRENT_VARIANT)_$(PLATFORM_PREFERRED_ARCH):dir:standardizepath:file:default=arm64)"
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
typealias OracleIdentifierType = String
|
||||
|
||||
/// Controls if the given type should be included or not
|
||||
/// Example: controls if remote cache integration should be added for a given target or configuration
|
||||
protocol IncludeOracle {
|
||||
/// Decides if a given type should be included or not
|
||||
/// - Parameter identifier: identifier of a type
|
||||
func shouldInclude(identifier: OracleIdentifierType) -> Bool
|
||||
}
|
||||
|
||||
struct IncludeExcludeOracle: IncludeOracle {
|
||||
let excludes: [OracleIdentifierType]
|
||||
let includes: [OracleIdentifierType]
|
||||
|
||||
|
||||
func shouldInclude(identifier: OracleIdentifierType) -> Bool {
|
||||
// exclude array has precedence.
|
||||
if excludes.contains(identifier) {
|
||||
return false
|
||||
}
|
||||
guard !includes.isEmpty else {
|
||||
return true
|
||||
}
|
||||
return includes.contains(identifier)
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Integrates XCRemoteCache into the existing Xcode project
|
||||
protocol Integrate {
|
||||
/// Entry point for the XCRemoteCache integration
|
||||
func run() throws
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
struct IntegrateContext {
|
||||
let projectPath: URL
|
||||
let repoRoot: URL
|
||||
let binaries: XCRCBinariesPaths
|
||||
let mode: Mode
|
||||
let configOverride: URL
|
||||
let fakeSrcRoot: URL
|
||||
let output: URL?
|
||||
}
|
||||
|
||||
extension IntegrateContext {
|
||||
init(
|
||||
input: String,
|
||||
repoRootPath: String,
|
||||
mode: Mode,
|
||||
configOverridePath: String,
|
||||
env: [String: String],
|
||||
binariesDir: URL,
|
||||
fakeSrcRoot: String,
|
||||
outputPath: String?
|
||||
) throws {
|
||||
projectPath = URL(fileURLWithPath: input)
|
||||
let srcRoot = projectPath.deletingLastPathComponent()
|
||||
repoRoot = URL(fileURLWithPath: repoRootPath, relativeTo: srcRoot)
|
||||
self.mode = mode
|
||||
configOverride = URL(fileURLWithPath: configOverridePath, relativeTo: srcRoot)
|
||||
output = outputPath.flatMap(URL.init(fileURLWithPath:))
|
||||
self.fakeSrcRoot = URL(fileURLWithPath: fakeSrcRoot)
|
||||
binaries = XCRCBinariesPaths(
|
||||
prepare: binariesDir.appendingPathComponent("xcprepare"),
|
||||
cc: binariesDir.appendingPathComponent("xccc"),
|
||||
swiftc: binariesDir.appendingPathComponent("xcswiftc"),
|
||||
libtool: binariesDir.appendingPathComponent("xclibtool"),
|
||||
ld: binariesDir.appendingPathComponent("xcld"),
|
||||
prebuild: binariesDir.appendingPathComponent("xcprebuild"),
|
||||
postbuild: binariesDir.appendingPathComponent("xcpostbuild")
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
public enum LLDBInitMode: String, Codable, CaseIterable {
|
||||
/// Do not add anything to .lldbinit (might affect debugging experience)
|
||||
case none
|
||||
/// Installs lldb command in a ~/.lldbinit
|
||||
case user
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Saves integration specific lldb command to the .lldbinit file
|
||||
protocol LLDBInitPatcher {
|
||||
func enable() throws
|
||||
}
|
||||
|
||||
/// Does nothing for patching
|
||||
class NoopLLDBInitPatcher: LLDBInitPatcher {
|
||||
func enable() throws {}
|
||||
}
|
||||
|
||||
// Saves a custom lldb command (XCRC lldb command) in a .lldbinit file with a preamble comment
|
||||
class FileLLDBInitPatcher: LLDBInitPatcher {
|
||||
/// A preamble string. A line after that string is managed by XCRemoteCache
|
||||
private static let preambleString = "#RemoteCacheCustomSourceMap"
|
||||
|
||||
private let fileLocation: URL
|
||||
private let lldbCommand: String
|
||||
private let fileAccessor: FileAccessor
|
||||
|
||||
/// Default initailizer
|
||||
/// - Parameters:
|
||||
/// - file: Location of the LLDB init file
|
||||
/// - rootURL: Root location of the LLDB target source-map
|
||||
/// - fakeSrcRoot: Arbitrary fake root location, shared between all producers and consumers
|
||||
/// - fileManager: fileManager
|
||||
init(
|
||||
file: URL,
|
||||
rootURL: URL,
|
||||
fakeSrcRoot: URL,
|
||||
fileAccessor: FileAccessor
|
||||
) {
|
||||
fileLocation = file
|
||||
lldbCommand = "settings set target.source-map \(fakeSrcRoot.path) \(rootURL.path)"
|
||||
self.fileAccessor = fileAccessor
|
||||
}
|
||||
|
||||
private func findIndices(in collection: [String], value: String) -> [Int] {
|
||||
collection.enumerated().reduce([]) { result, line -> [Int] in
|
||||
if line.element == Self.preambleString {
|
||||
return result + [line.offset]
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
// Appends XCRC lldb command to the specifies file
|
||||
// Note: Doesn't modify the file if it already contains a valid command
|
||||
func enable() throws {
|
||||
var finalLines: [String]
|
||||
let xcrcLLDBCommandArray = [Self.preambleString, lldbCommand]
|
||||
if let content = try? fileAccessor.contents(atPath: fileLocation.path) {
|
||||
let contentString = String(data: content, encoding: .utf8)!
|
||||
let originalContentLines = contentString.components(separatedBy: .newlines)
|
||||
var contentLines = originalContentLines
|
||||
let preambleIndices = findIndices(in: contentLines, value: Self.preambleString)
|
||||
|
||||
if !preambleIndices.isEmpty {
|
||||
let firstLLDBCommandIndex = preambleIndices[0] + 1
|
||||
if firstLLDBCommandIndex >= contentLines.count {
|
||||
// corrupted file, append the script line at the bottom
|
||||
contentLines.append(lldbCommand)
|
||||
} else {
|
||||
if preambleIndices.count == 1 && contentLines[firstLLDBCommandIndex] == lldbCommand {
|
||||
// the file content is already valid
|
||||
return
|
||||
}
|
||||
contentLines[firstLLDBCommandIndex] = lldbCommand
|
||||
}
|
||||
|
||||
// Delete excessive XCRC lldb commands
|
||||
for index in preambleIndices.dropFirst().reversed() {
|
||||
let rangeEnd = min(index + 1, contentLines.count - 1)
|
||||
contentLines.removeSubrange(index...rangeEnd)
|
||||
}
|
||||
} else {
|
||||
contentLines += xcrcLLDBCommandArray
|
||||
}
|
||||
finalLines = contentLines
|
||||
} else {
|
||||
finalLines = xcrcLLDBCommandArray
|
||||
}
|
||||
// Save to disk
|
||||
if finalLines.suffix(xcrcLLDBCommandArray.count) == xcrcLLDBCommandArray {
|
||||
// always end with empty line when appending a command at the bottom
|
||||
finalLines.append("")
|
||||
}
|
||||
let finalContent = finalLines.joined(separator: "\n").data(using: .utf8)
|
||||
try fileAccessor.write(toPath: fileLocation.path, contents: finalContent)
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public class XCIntegrate {
|
||||
/// Separator of sequential command line arguments (e.g. configurations to exclude)
|
||||
fileprivate static let inputSeparate: Character = ","
|
||||
|
||||
private let projectPath: String
|
||||
private let mode: Mode
|
||||
private let configurationsExclude: String
|
||||
private let configurationsInclude: String
|
||||
private let targetsExclude: String
|
||||
private let targetsInclude: String
|
||||
private let finalProducerTarget: String?
|
||||
private let consumerEligibleConfigurations: String
|
||||
private let consumerEligiblePlatforms: String
|
||||
private let lldbMode: LLDBInitMode
|
||||
private let fakeSrcRoot: String
|
||||
private let output: String?
|
||||
|
||||
public init(
|
||||
input: String,
|
||||
mode: Mode,
|
||||
configurationsExclude: String,
|
||||
configurationsInclude: String,
|
||||
targetsExclude: String,
|
||||
targetsInclude: String,
|
||||
finalProducerTarget: String?,
|
||||
consumerEligibleConfigurations: String,
|
||||
consumerEligiblePlatforms: String,
|
||||
lldbMode: LLDBInitMode,
|
||||
fakeSrcRoot: String,
|
||||
output: String?
|
||||
) {
|
||||
projectPath = input
|
||||
self.mode = mode
|
||||
self.configurationsExclude = configurationsExclude
|
||||
self.configurationsInclude = configurationsInclude
|
||||
self.targetsExclude = targetsExclude
|
||||
self.targetsInclude = targetsInclude
|
||||
self.finalProducerTarget = finalProducerTarget
|
||||
self.consumerEligibleConfigurations = consumerEligibleConfigurations
|
||||
self.consumerEligiblePlatforms = consumerEligiblePlatforms
|
||||
self.lldbMode = lldbMode
|
||||
self.fakeSrcRoot = fakeSrcRoot
|
||||
self.output = output
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
public func main() {
|
||||
do {
|
||||
let env = ProcessInfo.processInfo.environment
|
||||
let fileManager = FileManager.default
|
||||
let commandURL = URL(fileURLWithPath: ProcessInfo.processInfo.arguments[0])
|
||||
// All binaries (xcprepare, xcprebuild etc.) should be placed next to each other
|
||||
let binariesDir = commandURL.deletingLastPathComponent()
|
||||
|
||||
let srcRoot: URL = URL(fileURLWithPath: projectPath).deletingLastPathComponent()
|
||||
let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
|
||||
.readConfiguration()
|
||||
|
||||
let context = try IntegrateContext(
|
||||
input: projectPath,
|
||||
repoRootPath: config.repoRoot,
|
||||
mode: mode,
|
||||
configOverridePath: config.extraConfigurationFile,
|
||||
env: env,
|
||||
binariesDir: binariesDir,
|
||||
fakeSrcRoot: fakeSrcRoot,
|
||||
outputPath: output
|
||||
)
|
||||
let configurationOracle = IncludeExcludeOracle(
|
||||
excludes: configurationsExclude.integrateArrayArguments,
|
||||
includes: configurationsInclude.integrateArrayArguments
|
||||
)
|
||||
let targetOracle = IncludeExcludeOracle(
|
||||
excludes: targetsExclude.integrateArrayArguments,
|
||||
includes: targetsInclude.integrateArrayArguments
|
||||
)
|
||||
let buildSettingsAppender = XcodeProjBuildSettingsIntegrateAppender(
|
||||
mode: context.mode,
|
||||
repoRoot: context.repoRoot
|
||||
)
|
||||
let lldbPatcher: LLDBInitPatcher
|
||||
switch lldbMode {
|
||||
case .none:
|
||||
lldbPatcher = NoopLLDBInitPatcher()
|
||||
case .user:
|
||||
let lldbInitFile = URL(fileURLWithPath: "~/.lldbinit".expandingTildeInPath)
|
||||
lldbPatcher = FileLLDBInitPatcher(
|
||||
file: lldbInitFile,
|
||||
rootURL: context.repoRoot,
|
||||
fakeSrcRoot: context.fakeSrcRoot,
|
||||
fileAccessor: fileManager
|
||||
)
|
||||
}
|
||||
|
||||
let integrator = XcodeProjIntegrate(
|
||||
project: context.projectPath,
|
||||
mode: context.mode,
|
||||
binaries: context.binaries,
|
||||
configurationIncludeOracle: configurationOracle,
|
||||
targetIncludeOracle: targetOracle,
|
||||
finalProducerTarget: finalProducerTarget,
|
||||
buildSettingsAppender: buildSettingsAppender,
|
||||
consumerEligibleConfigurations: consumerEligibleConfigurations.integrateArrayArguments,
|
||||
consumerEligiblePlatforms: consumerEligiblePlatforms.integrateArrayArguments,
|
||||
configOverride: context.configOverride,
|
||||
lldbPatcher: lldbPatcher,
|
||||
output: context.output
|
||||
)
|
||||
try integrator.run()
|
||||
} catch {
|
||||
// XCIntegrate has no fallback
|
||||
exit(1, "FATAL: Integrate initialization failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private extension String {
|
||||
var integrateArrayArguments: [String] {
|
||||
split(separator: XCIntegrate.inputSeparate).map(String.init)
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Representing locations of all XCRemoteCache binaries (including wrappers and phase scripts)
|
||||
struct XCRCBinariesPaths {
|
||||
let prepare: URL
|
||||
let cc: URL
|
||||
let swiftc: URL
|
||||
let libtool: URL
|
||||
let ld: URL
|
||||
let prebuild: URL
|
||||
let postbuild: URL
|
||||
}
|
||||
@@ -1,249 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import PathKit
|
||||
import XcodeProj
|
||||
import Yams
|
||||
|
||||
enum XcodeProjIntegrateError: Error {
|
||||
/// Thrown when backend server doesn't contain a commit sha with all artifacts ready
|
||||
case noArtifactsToReuse
|
||||
}
|
||||
|
||||
/// Integrates XCRemoteCache using third-party XcodeProj library
|
||||
struct XcodeProjIntegrate: Integrate {
|
||||
fileprivate static let BuildStepPrefix = "[RC] "
|
||||
|
||||
// IntegrationConfiguration represents a subset of the XCRemoteCacheConfig configuration
|
||||
private struct IntegrationCacheConfig: Encodable {
|
||||
let recommendedCacheAddress: URL?
|
||||
let xcccFile: String
|
||||
let mode: Mode
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case xcccFile = "xccc_file"
|
||||
case recommendedCacheAddress = "recommended_cache_address"
|
||||
case mode
|
||||
}
|
||||
}
|
||||
|
||||
private let projectURL: URL
|
||||
private let mode: Mode
|
||||
private let binaries: XCRCBinariesPaths
|
||||
private let configurationIncludeOracle: IncludeOracle
|
||||
private let targetIncludeOracle: IncludeOracle
|
||||
private let finalProducerTarget: String?
|
||||
private let buildSettingsAppender: BuildSettingsIntegrateAppender
|
||||
private let consumerEligibleConfigurations: [String]
|
||||
private let consumerEligiblePlatforms: [String]
|
||||
private let prebuildPhase: PBXShellScriptBuildPhase
|
||||
private let postbuildPhase: PBXShellScriptBuildPhase
|
||||
private let markPhase: PBXShellScriptBuildPhase
|
||||
private let configOverride: URL
|
||||
private let lldbPatcher: LLDBInitPatcher
|
||||
private let output: URL?
|
||||
|
||||
init(
|
||||
project: URL,
|
||||
mode: Mode,
|
||||
binaries: XCRCBinariesPaths,
|
||||
configurationIncludeOracle: IncludeOracle,
|
||||
targetIncludeOracle: IncludeOracle,
|
||||
finalProducerTarget: String?,
|
||||
buildSettingsAppender: BuildSettingsIntegrateAppender,
|
||||
consumerEligibleConfigurations: [String],
|
||||
consumerEligiblePlatforms: [String],
|
||||
configOverride: URL,
|
||||
lldbPatcher: LLDBInitPatcher,
|
||||
output: URL?
|
||||
) {
|
||||
projectURL = project
|
||||
self.mode = mode
|
||||
self.binaries = binaries
|
||||
self.configurationIncludeOracle = configurationIncludeOracle
|
||||
self.targetIncludeOracle = targetIncludeOracle
|
||||
self.finalProducerTarget = finalProducerTarget
|
||||
self.buildSettingsAppender = buildSettingsAppender
|
||||
self.consumerEligibleConfigurations = consumerEligibleConfigurations
|
||||
self.consumerEligiblePlatforms = consumerEligiblePlatforms
|
||||
self.configOverride = configOverride
|
||||
self.lldbPatcher = lldbPatcher
|
||||
self.output = output
|
||||
|
||||
prebuildPhase = PBXShellScriptBuildPhase(
|
||||
name: "\(Self.BuildStepPrefix)RemoteCache_prebuild",
|
||||
inputPaths: [binaries.prebuild.path],
|
||||
outputPaths: ["$(TARGET_TEMP_DIR)/rc.enabled"],
|
||||
shellScript: "\"$SCRIPT_INPUT_FILE_0\"",
|
||||
dependencyFile: "$(TARGET_TEMP_DIR)/prebuild.d"
|
||||
)
|
||||
postbuildPhase = PBXShellScriptBuildPhase(
|
||||
name: "\(Self.BuildStepPrefix)RemoteCache_postbuild",
|
||||
inputPaths: [binaries.postbuild.path],
|
||||
outputPaths: [
|
||||
"""
|
||||
$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/\
|
||||
$(XCRC_PLATFORM_PREFERRED_ARCH).swiftmodule.md5
|
||||
""",
|
||||
"""
|
||||
$(TARGET_BUILD_DIR)/$(MODULES_FOLDER_PATH)/$(PRODUCT_MODULE_NAME).swiftmodule/\
|
||||
$(XCRC_PLATFORM_PREFERRED_ARCH)-$(LLVM_TARGET_TRIPLE_VENDOR)-$(SWIFT_PLATFORM_TARGET_PREFIX)\
|
||||
$(LLVM_TARGET_TRIPLE_SUFFIX).swiftmodule.md5
|
||||
""",
|
||||
],
|
||||
shellScript: "\"$SCRIPT_INPUT_FILE_0\"",
|
||||
dependencyFile: "$(TARGET_TEMP_DIR)/postbuild.d"
|
||||
)
|
||||
markPhase = PBXShellScriptBuildPhase(
|
||||
name: "\(Self.BuildStepPrefix)RemoteCache_mark",
|
||||
inputPaths: [binaries.prepare.path],
|
||||
shellScript:
|
||||
"\"$SCRIPT_INPUT_FILE_0\" mark " +
|
||||
"--configuration \"$CONFIGURATION\" --platform \"$PLATFORM_NAME\""
|
||||
)
|
||||
}
|
||||
|
||||
/// Dump overrides to the XCRemoteCacheConfig into disk location
|
||||
private func storeRCOverride(
|
||||
_ override: IntegrationCacheConfig,
|
||||
configOverrideLocation: URL
|
||||
) throws {
|
||||
// Store .rcinfo override
|
||||
let encoder = YAMLEncoder()
|
||||
let encodedYAML = try encoder.encode(override)
|
||||
try encodedYAML.write(to: configOverrideLocation, atomically: false, encoding: .utf8)
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
func run() throws {
|
||||
let outputFile = output ?? projectURL
|
||||
let projectRoot = projectURL.deletingLastPathComponent()
|
||||
|
||||
let projectPath = Path(projectURL.path)
|
||||
let outputPath = Path(outputFile.path)
|
||||
|
||||
// Override all extra configs (default to 'user.rc', next to the main '.rcinfo' file)
|
||||
let initialOverride = IntegrationCacheConfig(
|
||||
recommendedCacheAddress: nil,
|
||||
xcccFile: binaries.cc.path,
|
||||
mode: mode
|
||||
)
|
||||
try storeRCOverride(initialOverride, configOverrideLocation: configOverride)
|
||||
|
||||
if case .consumer = mode {
|
||||
// require successful preparation
|
||||
do {
|
||||
|
||||
// Call xcprepare to probe if XCRemoteCache can be safely used
|
||||
let args = ["--configuration"] + consumerEligibleConfigurations + ["--platform"] +
|
||||
consumerEligiblePlatforms
|
||||
let yamlString = try shellGetStdout(
|
||||
binaries.prepare.path,
|
||||
args: args,
|
||||
inDir: projectRoot.path,
|
||||
environment: nil
|
||||
)
|
||||
let decoder = YAMLDecoder()
|
||||
let prepareResult = try decoder.decode(PrepareResult.self, from: yamlString, userInfo: [:])
|
||||
guard case .preparedFor(_, recommendedCacheAddress: let remote) = prepareResult else {
|
||||
throw XcodeProjIntegrateError.noArtifactsToReuse
|
||||
}
|
||||
|
||||
// Override the configuration again to include recommended cache address provided by xcprepare
|
||||
let finalOverride = IntegrationCacheConfig(
|
||||
recommendedCacheAddress: remote,
|
||||
xcccFile: binaries.cc.path,
|
||||
mode: mode
|
||||
)
|
||||
try storeRCOverride(finalOverride, configOverrideLocation: configOverride)
|
||||
} catch {
|
||||
// integration cannot be done as `xccc` hasn't been generated
|
||||
exit(1, "XCRemoteCache cannot be initialized with a consumer mode. Error: \(error).")
|
||||
}
|
||||
}
|
||||
|
||||
// modify .pbxproj
|
||||
let xcodeproj = try XcodeProj(path: projectPath)
|
||||
|
||||
for target in xcodeproj.pbxproj.nativeTargets {
|
||||
guard targetIncludeOracle.shouldInclude(identifier: target.name) else {
|
||||
continue
|
||||
}
|
||||
guard let targetConfigurations = target.buildConfigurationList else {
|
||||
fatalError("Missing buildConfigurationList. Cannot apply")
|
||||
}
|
||||
|
||||
// Apply settings for only few configurations
|
||||
let targetConfigurationsToIntegrate = targetConfigurations.buildConfigurations.filter {
|
||||
configurationIncludeOracle.shouldInclude(identifier: $0.name)
|
||||
}
|
||||
|
||||
guard !targetConfigurationsToIntegrate.isEmpty else {
|
||||
// No need to append build phases if none of Configurations exist for that target
|
||||
continue
|
||||
}
|
||||
|
||||
for buildConfiguration in targetConfigurationsToIntegrate {
|
||||
let initialSettings = buildConfiguration.buildSettings
|
||||
let finalSettings = buildSettingsAppender.appendToBuildSettings(
|
||||
buildSettings: initialSettings,
|
||||
wrappers: binaries
|
||||
)
|
||||
buildConfiguration.buildSettings = finalSettings
|
||||
}
|
||||
|
||||
addSharedBuildPhases(target: target, in: xcodeproj.pbxproj)
|
||||
// Add producer build phase that marks given sha+configuration+platform as "ready to use"
|
||||
if case .producer = mode, finalProducerTarget == target.name {
|
||||
addFinalProducerBuildPhases(target: target, in: xcodeproj.pbxproj)
|
||||
}
|
||||
}
|
||||
|
||||
try xcodeproj.write(path: outputPath)
|
||||
|
||||
try lldbPatcher.enable()
|
||||
}
|
||||
|
||||
/// Adds build phases for both producer and consumer
|
||||
private func addSharedBuildPhases(target: PBXNativeTarget, in pbxproj: PBXProj) {
|
||||
// delete all previous XCRC build phases
|
||||
let previousRCPhases = target.buildPhases.filter(isRCPhase)
|
||||
target.buildPhases.removeAll(where: previousRCPhases.contains)
|
||||
|
||||
if let sourceIndex = target.buildPhases.map(\.buildPhase).firstIndex(of: .sources) {
|
||||
// add (pre|post)build phases only when a target has some compilation steps
|
||||
// otherwise they make no sense (nothing to store in an artifact)
|
||||
pbxproj.add(object: prebuildPhase)
|
||||
target.buildPhases.insert(prebuildPhase, at: sourceIndex)
|
||||
pbxproj.add(object: postbuildPhase)
|
||||
target.buildPhases.append(postbuildPhase)
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds build phases as the very last producer target
|
||||
private func addFinalProducerBuildPhases(target: PBXNativeTarget, in pbxproj: PBXProj) {
|
||||
pbxproj.add(object: markPhase)
|
||||
target.buildPhases.append(markPhase)
|
||||
}
|
||||
|
||||
private func isRCPhase(_ phase: PBXBuildPhase) -> Bool {
|
||||
phase.name()?.hasPrefix(Self.BuildStepPrefix) == true
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Type to manage Xcode build setting with compilation flags (e.g. OTHER_CFLAGS or OTHER_SWIFT_FLAGS)
|
||||
public protocol XcodeSettingsFlags {
|
||||
var settingValue: String? { get }
|
||||
|
||||
mutating func assignFlag(key: String, value: String?)
|
||||
}
|
||||
|
||||
/// Builds compilation flags string value
|
||||
private struct XcodeSettingsBuilder {
|
||||
static let inheritedExpression: String = "$(inherited)"
|
||||
|
||||
static func composeFlags(_ flags: [String]) -> String? {
|
||||
if flags == [Self.inheritedExpression] {
|
||||
return nil
|
||||
}
|
||||
return flags.joined(separator: " ")
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages flags for OTHER_SWIFT_FLAGS Xcode's Build Setting
|
||||
struct XcodeSettingsSwiftFlags: XcodeSettingsFlags {
|
||||
private static let swiftFlagPrefix = "-"
|
||||
|
||||
private(set) var settingValue: String?
|
||||
|
||||
init(settingValue: String?) {
|
||||
self.settingValue = settingValue
|
||||
}
|
||||
|
||||
private func buildSwiftFlag(key: String, value: String) -> [String] {
|
||||
[key, value]
|
||||
}
|
||||
|
||||
mutating func assignFlag(key: String, value: String?) {
|
||||
let flags: [String]
|
||||
let formattedKey = Self.swiftFlagPrefix.appending(key)
|
||||
switch (settingValue, value) {
|
||||
case (nil, nil):
|
||||
return
|
||||
case (nil, .some(let value)):
|
||||
flags = [XcodeSettingsBuilder.inheritedExpression, formattedKey, value]
|
||||
case (.some(let existing), _):
|
||||
var flagsComponents: [String] = existing.split(separator: " ").map(String.init)
|
||||
// remove (if exists)
|
||||
if let previousIndex = flagsComponents.firstIndex(of: formattedKey) {
|
||||
// delete "-{key}" and "{value}"
|
||||
flagsComponents.removeSubrange(previousIndex..<previousIndex + 2)
|
||||
}
|
||||
// add if setting a non nil value
|
||||
if let newValue = value {
|
||||
flagsComponents += buildSwiftFlag(key: formattedKey, value: newValue)
|
||||
}
|
||||
flags = flagsComponents
|
||||
}
|
||||
settingValue = XcodeSettingsBuilder.composeFlags(flags)
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages flags for OTHER_CFLAGS Xcode's Build Setting
|
||||
struct XcodeSettingsCFlags: XcodeSettingsFlags {
|
||||
private static let prefix = "-f"
|
||||
private(set) var settingValue: String?
|
||||
|
||||
init(settingValue: String?) {
|
||||
self.settingValue = settingValue
|
||||
}
|
||||
|
||||
private func buildCFlag(key: String, value: String) -> [String] {
|
||||
["\(Self.prefix)\(key)=\(value)"]
|
||||
}
|
||||
|
||||
mutating func assignFlag(key: String, value: String?) {
|
||||
let flags: [String]
|
||||
switch (settingValue, value) {
|
||||
case (nil, nil):
|
||||
return
|
||||
case (nil, .some(let value)):
|
||||
flags = [XcodeSettingsBuilder.inheritedExpression] + buildCFlag(key: key, value: value)
|
||||
case (.some(let existing), _):
|
||||
var flagsComponents: [String] = existing.split(separator: " ").map(String.init)
|
||||
// remove (if exists)
|
||||
let existingFlagIndex = flagsComponents.firstIndex { component -> Bool in
|
||||
component.hasPrefix("\(Self.prefix)\(key)=")
|
||||
}
|
||||
if let index = existingFlagIndex {
|
||||
flagsComponents.remove(at: index)
|
||||
}
|
||||
// add (if sets new)
|
||||
if let newValue = value {
|
||||
flagsComponents += buildCFlag(key: key, value: newValue)
|
||||
}
|
||||
flags = flagsComponents
|
||||
}
|
||||
settingValue = XcodeSettingsBuilder.composeFlags(flags)
|
||||
}
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum PrepareResult: Equatable {
|
||||
struct ShaInfo: Equatable, Encodable {
|
||||
/// Sha of the commit
|
||||
let sha: String
|
||||
/// Number of skipped commits to reach that sha from HEAD
|
||||
/// For a repo with a merge strategy - number of merge commits from HEAD
|
||||
let age: Int
|
||||
}
|
||||
|
||||
case preparedFor(sha: ShaInfo, recommendedCacheAddress: URL)
|
||||
case failed
|
||||
}
|
||||
|
||||
protocol PrepareLogic {
|
||||
func prepare() throws -> PrepareResult
|
||||
}
|
||||
|
||||
enum PrepareError: Error {
|
||||
/// Cannot find common commit sha with primary branch
|
||||
case invalidSha
|
||||
/// xcode-select does not specify current xcode
|
||||
case missingXcodeSelectDirectory
|
||||
}
|
||||
|
||||
class Prepare: PrepareLogic {
|
||||
|
||||
private let context: PrepareContext
|
||||
private let gitClient: GitClient
|
||||
private let networkClients: [RemoteNetworkClient]
|
||||
private let ccBuilder: CCWrapperBuilder
|
||||
private let fileAccessor: FileAccessor
|
||||
private let cacheInvalidator: CacheInvalidator
|
||||
private let globalCacheSwitcher: GlobalCacheSwitcher
|
||||
|
||||
init(
|
||||
context: PrepareContext,
|
||||
gitClient: GitClient,
|
||||
networkClients: [RemoteNetworkClient],
|
||||
ccBuilder: CCWrapperBuilder,
|
||||
fileAccessor: FileAccessor,
|
||||
globalCacheSwitcher: GlobalCacheSwitcher,
|
||||
cacheInvalidator: CacheInvalidator
|
||||
) {
|
||||
self.context = context
|
||||
self.gitClient = gitClient
|
||||
self.networkClients = networkClients
|
||||
self.ccBuilder = ccBuilder
|
||||
self.fileAccessor = fileAccessor
|
||||
self.cacheInvalidator = cacheInvalidator
|
||||
self.globalCacheSwitcher = globalCacheSwitcher
|
||||
}
|
||||
|
||||
/// Finds the best commit with generated artifacts to use
|
||||
func prepare() throws -> PrepareResult {
|
||||
do {
|
||||
guard fileAccessor.fileExists(atPath: PhaseCacheModeController.xcodeSelectLink.path) else {
|
||||
throw PrepareError.missingXcodeSelectDirectory
|
||||
}
|
||||
let commonSha = try gitClient.getCommonPrimarySha()
|
||||
|
||||
if context.offline {
|
||||
// Optimistically take first common sha
|
||||
return try enableCommit(sha: commonSha, age: 0)
|
||||
}
|
||||
// Remove old artifacts from local cache
|
||||
cacheInvalidator.invalidateArtifacts()
|
||||
|
||||
// calling `git` is expensive, so optimistically tring the common sha first
|
||||
if try isArtifactAvailable(for: commonSha) {
|
||||
return try enableCommit(sha: commonSha, age: 0)
|
||||
}
|
||||
// Find a list of all potential commits that may have artifacts that can be used
|
||||
let allCommonCommits = try gitClient.getPreviousCommits(starting: commonSha, maximum: context.maximumSha)
|
||||
// First commit was checked already
|
||||
for (index, sha) in allCommonCommits.dropFirst().enumerated() {
|
||||
// Check if the marker file for a `sha` commit is available on the remote cache server
|
||||
if try isArtifactAvailable(for: sha) {
|
||||
// adding 1 because current HEAD was already checked
|
||||
return try enableCommit(sha: sha, age: index + 1)
|
||||
}
|
||||
}
|
||||
infoLog("No artifacts available")
|
||||
try disable()
|
||||
} catch {
|
||||
try disable()
|
||||
throw error
|
||||
}
|
||||
return .failed
|
||||
}
|
||||
|
||||
private func isArtifactAvailable(for commit: String) throws -> Bool {
|
||||
try networkClients.allSatisfy { networkClient in
|
||||
try networkClient.fileExists(.marker(commit: commit))
|
||||
}
|
||||
}
|
||||
|
||||
private func enableCommit(sha: String, age: Int) throws -> PrepareResult {
|
||||
try globalCacheSwitcher.enable(sha: sha)
|
||||
try ccBuilder.compile(to: context.xcccCommand, commitSha: sha)
|
||||
return .preparedFor(sha: .init(sha: sha, age: age), recommendedCacheAddress: context.recommendedCacheAddress)
|
||||
}
|
||||
|
||||
private func disable() throws {
|
||||
try globalCacheSwitcher.disable()
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
|
||||
public enum PrepareContextError: Error {
|
||||
/// Provided primary repo is not a valid location
|
||||
case invalidPrimaryRepo(String)
|
||||
/// Provided primary branch is not a defined or invalid
|
||||
case invalidPrimaryBranch(String)
|
||||
/// Remote Cache server address is not valid URL
|
||||
case invalidRemoteCacheAddress(String)
|
||||
}
|
||||
|
||||
public struct PrepareContext {
|
||||
/// Path of the primary repository that produces cache artifacts
|
||||
let primaryRepo: String
|
||||
/// Main (primary) branch that produces cache artifacts
|
||||
let primaryBranch: String
|
||||
/// Path of the git repository
|
||||
let repoRoot: URL
|
||||
/// Location of the file that specifies remote commit sha
|
||||
let remoteCommitLocation: URL
|
||||
/// Maximum number of shas to look for a cache
|
||||
let maximumSha: Int
|
||||
/// skip making any HTTP requests and optimistically use a cache
|
||||
let offline: Bool
|
||||
/// Remote address of the remote server
|
||||
var recommendedCacheAddress: URL
|
||||
/// Remote addresses of all remote servers
|
||||
let cacheAddresses: [URL]
|
||||
/// Health path (relative to cacheAddresses) that determines request latency
|
||||
let cacheHealthPath: String
|
||||
/// Number of times to probe health path
|
||||
let cacheHealthPathProbeCount: Int
|
||||
/// clang wrapper output file
|
||||
let xcccCommand: URL
|
||||
}
|
||||
|
||||
extension PrepareContext {
|
||||
init(_ config: XCRemoteCacheConfig, offline: Bool) throws {
|
||||
guard !config.primaryRepo.isEmpty else {
|
||||
throw PrepareContextError.invalidPrimaryRepo(config.primaryRepo)
|
||||
}
|
||||
guard !config.primaryBranch.isEmpty else {
|
||||
throw PrepareContextError.invalidPrimaryBranch(config.primaryBranch)
|
||||
}
|
||||
primaryRepo = config.primaryRepo
|
||||
primaryBranch = config.primaryBranch
|
||||
let sourceRoot = URL(fileURLWithPath: config.sourceRoot, isDirectory: true)
|
||||
repoRoot = URL(fileURLWithPath: config.repoRoot, relativeTo: sourceRoot)
|
||||
remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: repoRoot)
|
||||
maximumSha = config.cacheCommitHistory
|
||||
self.offline = offline
|
||||
guard let address = URL(string: config.recommendedCacheAddress) else {
|
||||
throw PrepareContextError.invalidRemoteCacheAddress(config.recommendedCacheAddress)
|
||||
}
|
||||
recommendedCacheAddress = address
|
||||
xcccCommand = URL(fileURLWithPath: config.xcccFile, relativeTo: repoRoot)
|
||||
cacheAddresses = try config.cacheAddresses.map(URL.build)
|
||||
cacheHealthPath = config.cacheHealthPath
|
||||
cacheHealthPathProbeCount = config.cacheHealthPathProbeCount
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
|
||||
public enum PrepareMarkContextError: Error {
|
||||
case invalidAddress(String)
|
||||
}
|
||||
|
||||
public struct PrepareMarkContext {
|
||||
/// Path of the git repository
|
||||
let repoRoot: URL
|
||||
/// Remote address of the remote server
|
||||
let recommendedCacheAddress: URL
|
||||
/// All remote servers to mark
|
||||
let cacheAddresses: [URL]
|
||||
}
|
||||
|
||||
extension PrepareMarkContext {
|
||||
init(_ config: XCRemoteCacheConfig) throws {
|
||||
let sourceRoot = URL(fileURLWithPath: config.sourceRoot, isDirectory: true)
|
||||
repoRoot = URL(fileURLWithPath: config.repoRoot, relativeTo: sourceRoot)
|
||||
guard let address = URL(string: config.recommendedCacheAddress) else {
|
||||
errorLog("Invalid cache address: \(config.recommendedCacheAddress)")
|
||||
throw PrepareMarkContextError.invalidAddress(config.recommendedCacheAddress)
|
||||
}
|
||||
recommendedCacheAddress = address
|
||||
cacheAddresses = try config.cacheAddresses.map(URL.build)
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Print current configuration to the console
|
||||
public class XCConfig {
|
||||
private let outputEncoder: XCRemoteCacheEncoder
|
||||
|
||||
public init(format: XCOutputFormat) {
|
||||
outputEncoder = XCEncoderAbstractFactory().build(for: format)
|
||||
}
|
||||
|
||||
public func main() {
|
||||
let env = ProcessInfo.processInfo.environment
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
do {
|
||||
config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
|
||||
} catch {
|
||||
exit(1, "FATAL: Prepare initialization failed with error: \(error)")
|
||||
}
|
||||
|
||||
do {
|
||||
let output = try outputEncoder.encode(config)
|
||||
print(output)
|
||||
} catch {
|
||||
exit(1, "XCInfo failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Switch between Online/Offline modes
|
||||
public enum XCPrepareMode {
|
||||
/// Find the best common sha with primary with available marker on the remote cache server
|
||||
case online(configurations: [String], platforms: [String], customXcodeBuildNumber: String?)
|
||||
/// Skip making any HTTP requests and optimistically use a cache
|
||||
case offline
|
||||
}
|
||||
|
||||
/// 1) Finds the best sha to use with Remote Cache. Saves it to the local file and prints to the console
|
||||
/// 2) Compiles xccc wrapper from source
|
||||
/// 3) Invalidates outdated local cache entries
|
||||
public class XCPrepare {
|
||||
private let offline: Bool
|
||||
private let configurations: [String]
|
||||
private let platforms: [String]
|
||||
private let customXcodeBuildNumber: String?
|
||||
private let outputEncoder: XCRemoteCacheEncoder
|
||||
|
||||
public init(_ mode: XCPrepareMode, format: XCOutputFormat) {
|
||||
switch mode {
|
||||
case .offline:
|
||||
offline = true
|
||||
configurations = []
|
||||
platforms = []
|
||||
customXcodeBuildNumber = nil
|
||||
case .online(let configurations, let platforms, let customXcodeBuildNumber):
|
||||
offline = false
|
||||
self.platforms = platforms
|
||||
self.configurations = configurations
|
||||
self.customXcodeBuildNumber = customXcodeBuildNumber
|
||||
}
|
||||
outputEncoder = XCEncoderAbstractFactory().build(for: format)
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
public func main() {
|
||||
let env = ProcessInfo.processInfo.environment
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
var context: PrepareContext
|
||||
let xcodeVersion: String
|
||||
do {
|
||||
config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
|
||||
context = try PrepareContext(config, offline: offline)
|
||||
xcodeVersion = try customXcodeBuildNumber ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
|
||||
} catch {
|
||||
exit(1, "FATAL: Prepare initialization failed with error: \(error)")
|
||||
}
|
||||
|
||||
do {
|
||||
// TODO: Refactor to not pass empty arguments to `URLBuilderImpl`
|
||||
// URLs required by 'prepare' command are global for a project and don't required 'targetName'
|
||||
// or 'envFingerprint' - these are valid only for a target level requests
|
||||
let sessionFactory = DefaultURLSessionFactory(config: config)
|
||||
var awsV4Signature: AWSV4Signature?
|
||||
if !config.AWSAccessKey.isEmpty {
|
||||
awsV4Signature = AWSV4Signature(
|
||||
secretKey: config.AWSSecretKey,
|
||||
accessKey: config.AWSAccessKey,
|
||||
region: config.AWSRegion,
|
||||
service: config.AWSService,
|
||||
date: Date(timeIntervalSinceNow: 0)
|
||||
)
|
||||
}
|
||||
let networkClient = NetworkClientImpl(
|
||||
session: sessionFactory.build(),
|
||||
retries: config.downloadRetries,
|
||||
fileManager: fileManager,
|
||||
awsV4Signature: awsV4Signature
|
||||
)
|
||||
let serverProbe = try LowestLatencyNetworkServerProbe(
|
||||
servers: context.cacheAddresses,
|
||||
healthPath: context.cacheHealthPath,
|
||||
probes: context.cacheHealthPathProbeCount,
|
||||
fallbackServer: context.recommendedCacheAddress,
|
||||
networkClient: networkClient
|
||||
)
|
||||
context.recommendedCacheAddress = try serverProbe.determineRemoteServer()
|
||||
var networkClients: [RemoteNetworkClient] = []
|
||||
for platform in platforms {
|
||||
for configuration in configurations {
|
||||
let urlBuilder = URLBuilderImpl(
|
||||
address: context.recommendedCacheAddress,
|
||||
configuration: configuration,
|
||||
platform: platform,
|
||||
targetName: "",
|
||||
xcode: xcodeVersion,
|
||||
envFingerprint: "",
|
||||
schemaVersion: config.schemaVersion
|
||||
)
|
||||
networkClients.append(RemoteNetworkClientImpl(networkClient, urlBuilder))
|
||||
}
|
||||
}
|
||||
let primaryGitBranch = GitBranch(repoLocation: context.primaryRepo, branch: context.primaryBranch)
|
||||
let gitClient = GitClientImpl(
|
||||
repoRoot: context.repoRoot.path,
|
||||
primary: primaryGitBranch,
|
||||
shell: shellGetStdout
|
||||
)
|
||||
let ccBuilder = TemplateBasedCCWrapperBuilder(
|
||||
clangCommand: config.clangCommand,
|
||||
markerPath: config.modeMarkerPath,
|
||||
cachedTargetMockFilename: config.thinTargetMockFilename,
|
||||
prebuildDFilename: config.prebuildDiscoveryPath,
|
||||
compilationHistoryFilename: config.compilationHistoryFile,
|
||||
shellOut: shellGetStdout,
|
||||
fileManager: fileManager
|
||||
)
|
||||
let cacheURL: URL = fileManager.urls(for: .cachesDirectory, in: .userDomainMask).first!
|
||||
let localBuilder = LocalURLBuilderImpl(cachePath: cacheURL)
|
||||
let cacheAddress = localBuilder.location(for: context.recommendedCacheAddress)
|
||||
let cacheInvalidator = LocalCacheInvalidator(
|
||||
localCacheURL: cacheAddress,
|
||||
maximumAgeInDays: config.artifactMaximumAge
|
||||
)
|
||||
let fileAccessor = LazyFileAccessor(fileAccessor: FileManager.default)
|
||||
let globalCacheSwitcher = FileGlobalCacheSwitcher(context.remoteCommitLocation, fileAccessor: fileAccessor)
|
||||
|
||||
let prepare = Prepare(
|
||||
context: context,
|
||||
gitClient: gitClient,
|
||||
networkClients: networkClients,
|
||||
ccBuilder: ccBuilder,
|
||||
fileAccessor: fileAccessor,
|
||||
globalCacheSwitcher: globalCacheSwitcher,
|
||||
cacheInvalidator: cacheInvalidator
|
||||
)
|
||||
let prepareResult = try prepare.prepare()
|
||||
try outputResult(prepareResult)
|
||||
} catch GitClientError.missingPrimaryRepo(let repo) {
|
||||
exit(1, """
|
||||
XCRemoteCache's `xcprepare` failed to find git remote with \(repo) address.\
|
||||
Check that your git configuration (`git remote -v`) specifies it.
|
||||
""")
|
||||
} catch {
|
||||
exit(1, "Prepare failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Prints to the standard output, result of the prepare command
|
||||
private func outputResult(_ result: PrepareResult) throws {
|
||||
let outputString = try outputEncoder.encode(result)
|
||||
print(outputString)
|
||||
}
|
||||
}
|
||||
|
||||
extension PrepareResult: Encodable {
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case result
|
||||
case commit
|
||||
case age
|
||||
case recommendedRemoteAddress = "recommended_remote_address"
|
||||
}
|
||||
|
||||
func encode(to encoder: Encoder) throws {
|
||||
var container = encoder.container(keyedBy: CodingKeys.self)
|
||||
let result: Bool
|
||||
let commit: String?
|
||||
let age: Int?
|
||||
let recommendedRemoteAddress: URL?
|
||||
switch self {
|
||||
case .failed:
|
||||
result = false
|
||||
commit = nil
|
||||
age = nil
|
||||
recommendedRemoteAddress = nil
|
||||
case .preparedFor(let sha, let remoteAddress):
|
||||
result = true
|
||||
commit = sha.sha
|
||||
age = sha.age
|
||||
recommendedRemoteAddress = remoteAddress
|
||||
}
|
||||
try container.encode(result, forKey: .result)
|
||||
try container.encode(commit, forKey: .commit)
|
||||
try container.encode(age, forKey: .age)
|
||||
try container.encode(recommendedRemoteAddress, forKey: .recommendedRemoteAddress)
|
||||
}
|
||||
}
|
||||
|
||||
extension PrepareResult: Decodable {
|
||||
init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: Self.CodingKeys)
|
||||
let result = try container.decode(Bool.self, forKey: .result)
|
||||
if result {
|
||||
let commit = try container.decode(String.self, forKey: .commit)
|
||||
let age = try container.decode(Int.self, forKey: .age)
|
||||
let recommendedRemoteAddress = try container.decode(URL.self, forKey: .recommendedRemoteAddress)
|
||||
self = .preparedFor(sha: ShaInfo(sha: commit, age: age), recommendedCacheAddress: recommendedRemoteAddress)
|
||||
} else {
|
||||
self = .failed
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Marks current sha as artifact-available on the remote side
|
||||
public class XCPrepareMark {
|
||||
private let configuration: String
|
||||
private let platform: String
|
||||
private let xcode: String?
|
||||
private let commit: String?
|
||||
|
||||
public init(
|
||||
configuration: String,
|
||||
platform: String,
|
||||
xcode: String?,
|
||||
commit: String?
|
||||
) {
|
||||
self.configuration = configuration
|
||||
self.platform = platform
|
||||
self.xcode = xcode
|
||||
self.commit = commit
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
public func main() {
|
||||
let env = ProcessInfo.processInfo.environment
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
let context: PrepareMarkContext
|
||||
let xcodeVersion: String
|
||||
do {
|
||||
config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
|
||||
context = try PrepareMarkContext(config)
|
||||
xcodeVersion = try xcode ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
|
||||
} catch {
|
||||
exit(1, "FATAL: Prepare initialization failed with error: \(error)")
|
||||
}
|
||||
|
||||
do {
|
||||
let sessionFactory = DefaultURLSessionFactory(config: config)
|
||||
var awsV4Signature: AWSV4Signature?
|
||||
if !config.AWSAccessKey.isEmpty {
|
||||
awsV4Signature = AWSV4Signature(
|
||||
secretKey: config.AWSSecretKey,
|
||||
accessKey: config.AWSAccessKey,
|
||||
region: config.AWSRegion,
|
||||
service: config.AWSService,
|
||||
date: Date(timeIntervalSinceNow: 0)
|
||||
)
|
||||
}
|
||||
let networkClient = NetworkClientImpl(
|
||||
session: sessionFactory.build(),
|
||||
retries: config.uploadRetries,
|
||||
fileManager: fileManager,
|
||||
awsV4Signature: awsV4Signature
|
||||
)
|
||||
let remoteNetworkClient = try RemoteNetworkClientAbstractFactory(
|
||||
mode: .producer,
|
||||
downloadStreamURL: context.recommendedCacheAddress,
|
||||
upstreamStreamURL: context.cacheAddresses,
|
||||
networkClient: networkClient
|
||||
) { [configuration, platform] cacheAddress in
|
||||
// Prepare URLs don't include target name or envFingperint, which are valid only for a target level
|
||||
return URLBuilderImpl(
|
||||
address: cacheAddress,
|
||||
configuration: configuration,
|
||||
platform: platform,
|
||||
targetName: "",
|
||||
xcode: xcodeVersion,
|
||||
envFingerprint: "",
|
||||
schemaVersion: config.schemaVersion
|
||||
)
|
||||
}.build()
|
||||
|
||||
let gitCommit = try getCommitToMark(context: context, config: config)
|
||||
try remoteNetworkClient.createSynchronously(.marker(commit: gitCommit))
|
||||
} catch {
|
||||
exit(1, "Prepare failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
private func getCommitToMark(context: PrepareMarkContext, config: XCRemoteCacheConfig) throws -> String {
|
||||
if let commit = commit {
|
||||
return commit
|
||||
}
|
||||
let gitClient = GitClientImpl(
|
||||
repoRoot: context.repoRoot.path,
|
||||
primary: GitBranch(repoLocation: config.primaryRepo, branch: config.primaryBranch),
|
||||
shell: shellGetStdout
|
||||
)
|
||||
return try gitClient.getCurrentSha()
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Manages XCRemoteCache statistics: rests, print to the standard output etc
|
||||
public class XCStats {
|
||||
private let outputEncoder: XCRemoteCacheEncoder
|
||||
private let reset: Bool
|
||||
|
||||
public init(format: XCOutputFormat, reset: Bool) {
|
||||
self.reset = reset
|
||||
|
||||
outputEncoder = XCEncoderAbstractFactory().build(for: format)
|
||||
}
|
||||
|
||||
public func main() {
|
||||
let env = ProcessInfo.processInfo.environment
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
let context: XCStatsContext
|
||||
do {
|
||||
config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
|
||||
try context = XCStatsContext(config, fileManager: fileManager)
|
||||
} catch {
|
||||
exit(1, "FATAL: Prepare initialization failed with error: \(error)")
|
||||
}
|
||||
|
||||
do {
|
||||
let counterFactory: FileStatsCoordinator.CountersFactory = { file, count in
|
||||
ExclusiveFileCounter(ExclusiveFile(file, mode: .override), countersCount: count)
|
||||
}
|
||||
let statsCoordinator = try FileStatsCoordinator(
|
||||
statsLocation: context.statsDir,
|
||||
cacheLocationDir: context.cacheLocation,
|
||||
counterFactory: counterFactory,
|
||||
fileManager: fileManager
|
||||
)
|
||||
if reset {
|
||||
try statsCoordinator.reset()
|
||||
}
|
||||
let stats = try statsCoordinator.readStats()
|
||||
let output = try outputEncoder.encode(stats)
|
||||
print(output)
|
||||
} catch {
|
||||
exit(1, "XCStats failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
|
||||
public enum XCStatsContextError: Error {
|
||||
case invalidAddress(String)
|
||||
}
|
||||
|
||||
public struct XCStatsContext {
|
||||
/// Path of the root directory with all statistic files
|
||||
let statsDir: URL
|
||||
/// Location of the local cache that stores all fetched artifacts, metas etc
|
||||
let cacheLocation: URL
|
||||
}
|
||||
|
||||
extension XCStatsContext {
|
||||
init(_ config: XCRemoteCacheConfig, fileManager: FileManager) throws {
|
||||
|
||||
statsDir = URL(fileURLWithPath: config.statsDir.expandingTildeInPath)
|
||||
let cacheURL: URL = fileManager.urls(for: .cachesDirectory, in: .userDomainMask).first!
|
||||
let cacheURLBuilder = LocalURLBuilderImpl(cachePath: cacheURL)
|
||||
cacheLocation = cacheURLBuilder.localAddress
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Wrapper for `libtool` or `ld` call for creating a binary (a static or a dynamic library)
|
||||
/// Moves binary from a cache aritfact to the output location or
|
||||
/// fallbacks to the standard command (when cached product is not applicable)
|
||||
public class XCCreateBinary {
|
||||
private let output: URL
|
||||
private let tempDir: URL
|
||||
private let dependencyInfo: URL
|
||||
private let fallbackCommand: String
|
||||
private let stepDescription: String
|
||||
|
||||
/// Initializer of a binary creator step
|
||||
/// - Parameters:
|
||||
/// - output: Destination of the binary to create
|
||||
/// - filelist: location of a filelist file with all input files of that step
|
||||
/// - dependencyInfo: location of the file to specify all dependencies of that step
|
||||
/// - fallbackCommand: command of the fallback command
|
||||
/// - stepDescription: descriptive name of the step
|
||||
public init(
|
||||
output: String,
|
||||
filelist: String,
|
||||
dependencyInfo: String,
|
||||
fallbackCommand: String,
|
||||
stepDescription: String
|
||||
) {
|
||||
self.output = URL(fileURLWithPath: output)
|
||||
self.dependencyInfo = URL(fileURLWithPath: dependencyInfo)
|
||||
// fileList is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.LinkFileList
|
||||
// TODO: find better (stable) technique to determine `$TARGET_TEMP_DIR`
|
||||
tempDir = URL(fileURLWithPath: filelist)
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
self.fallbackCommand = fallbackCommand
|
||||
self.stepDescription = stepDescription
|
||||
}
|
||||
|
||||
private func fallbackToDefault() -> Never {
|
||||
let args = ProcessInfo().arguments
|
||||
let paramList = [fallbackCommand] + args.dropFirst()
|
||||
let cargs = paramList.map { strdup($0) } + [nil]
|
||||
execvp(fallbackCommand, cargs)
|
||||
|
||||
/// C-function `execv` returns only when the command fails
|
||||
exit(1)
|
||||
}
|
||||
|
||||
public func run() {
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
do {
|
||||
let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
|
||||
config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
|
||||
.readConfiguration()
|
||||
} catch {
|
||||
errorLog("\(stepDescription) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)")
|
||||
fallbackToDefault()
|
||||
}
|
||||
let markerURL = tempDir.appendingPathComponent(config.modeMarkerPath)
|
||||
do {
|
||||
let organizer = ZipArtifactOrganizer(targetTempDir: tempDir, fileManager: fileManager)
|
||||
let dependenciesWriter = FileDatWriter(dependencyInfo, fileManager: fileManager)
|
||||
let markerReader = FileMarkerReader(markerURL, fileManager: fileManager)
|
||||
guard fileManager.fileExists(atPath: markerURL.path) else {
|
||||
fallbackToDefault()
|
||||
}
|
||||
|
||||
let cachedArtifactDir = organizer.getActiveArtifactLocation()
|
||||
let outputFilename = output.lastPathComponent
|
||||
let cachedBinaryURL = cachedArtifactDir.appendingPathComponent(outputFilename)
|
||||
try fileManager.spt_forceLinkItem(at: cachedBinaryURL, to: output)
|
||||
try dependenciesWriter.enable(dependencies: markerReader.listFilesURLs(), outputs: [output])
|
||||
} catch {
|
||||
errorLog("\(stepDescription) failed with error: \(error). Fallbacking to \(fallbackCommand)")
|
||||
do {
|
||||
try fileManager.removeItem(at: markerURL)
|
||||
fallbackToDefault()
|
||||
} catch {
|
||||
exit(1, "FATAL: \(stepDescription) failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum MirroredLinkingSwiftcProductsGeneratorError: Error {
|
||||
/// When the generation source list misses a path to the main "swiftmodule" file
|
||||
case missingMainSwiftmoduleFileToGenerateFrom
|
||||
}
|
||||
|
||||
/// Products generator that finds swift products destination based on the artifact dir structure. It uses
|
||||
/// `LinkingSwiftcProductsGenerator` under the hood
|
||||
///
|
||||
/// Useful for cases where destination locations are not provided explicitly (e.g. in a thin projects)
|
||||
class MirroredLinkingSwiftcProductsGenerator: SwiftcProductsGenerator {
|
||||
private let arch: String
|
||||
private let buildDir: URL
|
||||
private let headersDir: URL
|
||||
private let diskCopier: DiskCopier
|
||||
|
||||
/// Default initializer
|
||||
/// - Parameters:
|
||||
/// - arch: architecture of the build
|
||||
/// - buildDir: directory where all *.swiftmodule products should be placed
|
||||
/// - headersDir: directory where generated ObjC headers should be placed
|
||||
/// - fileManager: fileManager instance
|
||||
init(
|
||||
arch: String,
|
||||
buildDir: URL,
|
||||
headersDir: URL,
|
||||
diskCopier: DiskCopier
|
||||
) {
|
||||
self.arch = arch
|
||||
self.buildDir = buildDir
|
||||
self.headersDir = headersDir
|
||||
self.diskCopier = diskCopier
|
||||
}
|
||||
|
||||
func generateFrom(
|
||||
artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
|
||||
artifactSwiftModuleObjCFile: URL
|
||||
) throws -> URL {
|
||||
/// Predict moduleName from the `*.swiftmodule` artifact
|
||||
let foundSwiftmoduleFile = artifactSwiftModuleFiles[.swiftmodule]
|
||||
guard let mainSwiftmoduleFile = foundSwiftmoduleFile else {
|
||||
throw MirroredLinkingSwiftcProductsGeneratorError.missingMainSwiftmoduleFileToGenerateFrom
|
||||
}
|
||||
let moduleName = mainSwiftmoduleFile.deletingPathExtension().lastPathComponent
|
||||
let modulePathOutput = buildDir
|
||||
.appendingPathComponent("\(moduleName).swiftmodule")
|
||||
.appendingPathComponent(arch)
|
||||
.appendingPathExtension("swiftmodule")
|
||||
let objcHeaderOutput = headersDir.appendingPathComponent("\(moduleName)-Swift.h")
|
||||
|
||||
let generator = DiskSwiftcProductsGenerator(
|
||||
modulePathOutput: modulePathOutput,
|
||||
objcHeaderOutput: objcHeaderOutput,
|
||||
diskCopier: diskCopier
|
||||
)
|
||||
|
||||
return try generator.generateFrom(
|
||||
artifactSwiftModuleFiles: artifactSwiftModuleFiles,
|
||||
artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum SwiftCResult {
|
||||
/// Swiftc mock cannot be used and fallback to the compilation is required
|
||||
case forceFallback
|
||||
/// All compilation steps were mocked correctly
|
||||
case success
|
||||
}
|
||||
|
||||
/// Swiftc mocking compilation
|
||||
protocol SwiftcProtocol {
|
||||
/// Tries to performs mocked compilation (moving all cached files to the expected location)
|
||||
/// If cached compilation products are not valid or incompatible, fallbacks to build-from-source
|
||||
/// - Returns: `.forceFallback` if the cached compilation products are incompatible and fallback
|
||||
/// to a standard 'swiftc' is required, `.success` otherwise
|
||||
/// - Throws: An error if there was an unrecoverable, serious error (e.g. IO error)
|
||||
func mockCompilation() throws -> SwiftCResult
|
||||
}
|
||||
|
||||
/// Swiftc wrapper that mocks compilation with noop and moves all expected products from cache location
|
||||
class Swiftc: SwiftcProtocol {
|
||||
/// Reader of all input files of the compilation
|
||||
private let inputFileListReader: ListReader
|
||||
/// Reader of the marker file lists - list of dependencies to set for swiftc compilation
|
||||
private let markerReader: ListReader
|
||||
/// Checks if the input file exists in the file list
|
||||
private let allowedFilesListScanner: FileListScanner
|
||||
/// Manager of the downloaded artifact package
|
||||
private let artifactOrganizer: ArtifactOrganizer
|
||||
/// Reads all input and output files for the compilation from an input filemap
|
||||
private let inputFilesReader: SwiftcInputReader
|
||||
/// Write manager of the marker file
|
||||
private let markerWriter: MarkerWriter
|
||||
/// Generates products at the desired destination
|
||||
private let productsGenerator: SwiftcProductsGenerator
|
||||
private let context: SwiftcContext
|
||||
private let fileManager: FileManager
|
||||
private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter
|
||||
private let touchFactory: (URL, FileManager) -> Touch
|
||||
private let plugins: [SwiftcProductGenerationPlugin]
|
||||
|
||||
init(
|
||||
inputFileListReader: ListReader,
|
||||
markerReader: ListReader,
|
||||
allowedFilesListScanner: FileListScanner,
|
||||
artifactOrganizer: ArtifactOrganizer,
|
||||
inputReader: SwiftcInputReader,
|
||||
context: SwiftcContext,
|
||||
markerWriter: MarkerWriter,
|
||||
productsGenerator: SwiftcProductsGenerator,
|
||||
fileManager: FileManager,
|
||||
dependenciesWriterFactory: @escaping (URL, FileManager) -> DependenciesWriter,
|
||||
touchFactory: @escaping (URL, FileManager) -> Touch,
|
||||
plugins: [SwiftcProductGenerationPlugin]
|
||||
) {
|
||||
self.inputFileListReader = inputFileListReader
|
||||
self.markerReader = markerReader
|
||||
self.allowedFilesListScanner = allowedFilesListScanner
|
||||
self.artifactOrganizer = artifactOrganizer
|
||||
inputFilesReader = inputReader
|
||||
self.context = context
|
||||
self.markerWriter = markerWriter
|
||||
self.productsGenerator = productsGenerator
|
||||
self.fileManager = fileManager
|
||||
self.dependenciesWriterFactory = dependenciesWriterFactory
|
||||
self.touchFactory = touchFactory
|
||||
self.plugins = plugins
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
func mockCompilation() throws -> SwiftCResult {
|
||||
let rcModeEnabled = markerReader.canRead()
|
||||
guard rcModeEnabled else {
|
||||
infoLog("Swiftc marker doesn't exist")
|
||||
return .forceFallback
|
||||
}
|
||||
|
||||
let inputFilesInputs = try inputFileListReader.listFilesURLs()
|
||||
let markerAllowedFiles = try markerReader.listFilesURLs()
|
||||
let cachedDependenciesWriterFactory = CachedFileDependenciesWriterFactory(
|
||||
dependencies: markerAllowedFiles,
|
||||
fileManager: fileManager,
|
||||
writerFactory: dependenciesWriterFactory
|
||||
)
|
||||
// Verify all input files to be present in a marker fileList
|
||||
let disallowedInputs = try inputFilesInputs.filter { try !allowedFilesListScanner.contains($0) }
|
||||
|
||||
if !disallowedInputs.isEmpty {
|
||||
// New file (disallowedFile) added without modifying the rest of the feature. Fallback to swiftc and
|
||||
// ensure that compilation from source will be forced up until next merge/rebase with "primary" branch
|
||||
infoLog("Swiftc new input file \(disallowedInputs)")
|
||||
// Deleting marker to indicate that the remote cached artifact cannot be used
|
||||
try markerWriter.disable()
|
||||
|
||||
// Save custom prebuild discovery content to make sure that the following prebuild
|
||||
// phase will not try to reuse cached artifact (if present)
|
||||
// In other words: let prebuild know that it should not try to reenable cache
|
||||
// until the next merge with primary
|
||||
switch context.mode {
|
||||
case .consumer(commit: .available(let remoteCommit)):
|
||||
let prebuildDiscoveryURL = context.tempDir.appendingPathComponent(context.prebuildDependenciesPath)
|
||||
let prebuildDiscoverWriter = dependenciesWriterFactory(prebuildDiscoveryURL, fileManager)
|
||||
try prebuildDiscoverWriter.write(skipForSha: remoteCommit)
|
||||
case .consumer, .producer, .producerFast:
|
||||
// Never skip prebuild phase and fallback to the swiftc compilation for:
|
||||
// 1) Not enabled remote cache, 2) producer(s)
|
||||
break
|
||||
}
|
||||
return .forceFallback
|
||||
}
|
||||
|
||||
let artifactLocation = artifactOrganizer.getActiveArtifactLocation()
|
||||
|
||||
// Read swiftmodule location from XCRemoteCache
|
||||
// arbitrary format swiftmodule/${arch}/${moduleName}.swift{module|doc|sourceinfo}
|
||||
let moduleName = context.modulePathOutput.deletingPathExtension().lastPathComponent
|
||||
let allCompilations = try inputFilesReader.read()
|
||||
let artifactSwiftmoduleDir = artifactLocation
|
||||
.appendingPathComponent("swiftmodule")
|
||||
.appendingPathComponent(context.arch)
|
||||
let artifactSwiftmoduleBase = artifactSwiftmoduleDir.appendingPathComponent(moduleName)
|
||||
let artifactSwiftmoduleFiles = Dictionary(
|
||||
uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions
|
||||
.map { ext, _ in
|
||||
(ext, artifactSwiftmoduleBase.appendingPathExtension(ext.rawValue))
|
||||
}
|
||||
)
|
||||
|
||||
// Build -Swift.h location from XCRemoteCache arbitrary format include/${arch}/${target}-Swift.h
|
||||
let artifactSwiftModuleObjCDir = artifactLocation
|
||||
.appendingPathComponent("include")
|
||||
.appendingPathComponent(context.arch)
|
||||
.appendingPathComponent(context.moduleName)
|
||||
// Move cached xxxx-Swift.h to the location passed in arglist
|
||||
// Alternatively, artifactSwiftModuleObjCFile could be built as a first .h file in artifactSwiftModuleObjCDir
|
||||
let artifactSwiftModuleObjCFile = artifactSwiftModuleObjCDir
|
||||
.appendingPathComponent(context.objcHeaderOutput.lastPathComponent)
|
||||
|
||||
_ = try productsGenerator.generateFrom(
|
||||
artifactSwiftModuleFiles: artifactSwiftmoduleFiles,
|
||||
artifactSwiftModuleObjCFile: artifactSwiftModuleObjCFile
|
||||
)
|
||||
|
||||
try plugins.forEach {
|
||||
try $0.generate(for: allCompilations)
|
||||
}
|
||||
|
||||
// Save individual .d and touch .o for each .swift file
|
||||
for compilation in allCompilations.files {
|
||||
if let object = compilation.object {
|
||||
// Touching .o is required to invalidate already existing .a or linked library
|
||||
let touch = touchFactory(object, fileManager)
|
||||
try touch.touch()
|
||||
}
|
||||
if let individualDeps = compilation.dependencies {
|
||||
// swiftc product should be invalidated if any of dependencies file has changed
|
||||
try cachedDependenciesWriterFactory.generate(output: individualDeps)
|
||||
}
|
||||
}
|
||||
// Save .d for the entire module
|
||||
try cachedDependenciesWriterFactory.generate(output: allCompilations.info.swiftDependencies)
|
||||
// Generate .d file with all deps in the "-master.d" (e.g. for WMO)
|
||||
if let wmoDeps = allCompilations.info.dependencies {
|
||||
try cachedDependenciesWriterFactory.generate(output: wmoDeps)
|
||||
}
|
||||
infoLog("Swiftc noop for \(context.target)")
|
||||
return .success
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct SwiftcContext {
|
||||
enum SwiftcMode: Equatable {
|
||||
case producer
|
||||
/// Commit sha of the commit to use during remote cache
|
||||
case consumer(commit: RemoteCommitInfo)
|
||||
/// Remote artifact exists and can be optimistically used in place of a local compilation
|
||||
case producerFast
|
||||
}
|
||||
|
||||
let objcHeaderOutput: URL
|
||||
let moduleName: String
|
||||
let modulePathOutput: URL
|
||||
/// File that defines output files locations (.d, .swiftmodule etc.)
|
||||
let filemap: URL
|
||||
let target: String
|
||||
/// File that contains input files for the swift module compilation
|
||||
let fileList: URL
|
||||
let tempDir: URL
|
||||
let arch: String
|
||||
let prebuildDependenciesPath: String
|
||||
let mode: SwiftcMode
|
||||
/// File that stores all compilation invocation arguments
|
||||
let invocationHistoryFile: URL
|
||||
|
||||
|
||||
public init(
|
||||
config: XCRemoteCacheConfig,
|
||||
objcHeaderOutput: String,
|
||||
moduleName: String,
|
||||
modulePathOutput: String,
|
||||
filemap: String,
|
||||
target: String,
|
||||
fileList: String
|
||||
) throws {
|
||||
self.objcHeaderOutput = URL(fileURLWithPath: objcHeaderOutput)
|
||||
self.moduleName = moduleName
|
||||
self.modulePathOutput = URL(fileURLWithPath: modulePathOutput)
|
||||
self.filemap = URL(fileURLWithPath: filemap)
|
||||
self.target = target
|
||||
self.fileList = URL(fileURLWithPath: fileList)
|
||||
// modulePathOutput is place in $TARGET_TEMP_DIR/Objects-normal/$ARCH/$TARGET_NAME.swiftmodule
|
||||
// That may be subject to change for other Xcode versions
|
||||
tempDir = URL(fileURLWithPath: modulePathOutput)
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
.deletingLastPathComponent()
|
||||
arch = URL(fileURLWithPath: modulePathOutput).deletingLastPathComponent().lastPathComponent
|
||||
|
||||
let srcRoot: URL = URL(fileURLWithPath: config.sourceRoot)
|
||||
let remoteCommitLocation = URL(fileURLWithPath: config.remoteCommitFile, relativeTo: srcRoot)
|
||||
prebuildDependenciesPath = config.prebuildDiscoveryPath
|
||||
switch config.mode {
|
||||
case .consumer:
|
||||
let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim())
|
||||
mode = .consumer(commit: remoteCommit)
|
||||
case .producer:
|
||||
mode = .producer
|
||||
case .producerFast:
|
||||
let remoteCommit = RemoteCommitInfo(try? String(contentsOf: remoteCommitLocation).trim())
|
||||
switch remoteCommit {
|
||||
case .unavailable:
|
||||
mode = .producer
|
||||
case .available:
|
||||
mode = .producerFast
|
||||
}
|
||||
}
|
||||
invocationHistoryFile = URL(fileURLWithPath: config.compilationHistoryFile, relativeTo: tempDir)
|
||||
}
|
||||
|
||||
init(
|
||||
config: XCRemoteCacheConfig,
|
||||
input: SwiftcArgInput
|
||||
) throws {
|
||||
try self.init(
|
||||
config: config,
|
||||
objcHeaderOutput: input.objcHeaderOutput,
|
||||
moduleName: input.moduleName,
|
||||
modulePathOutput: input.modulePathOutput,
|
||||
filemap: input.filemap,
|
||||
target: input.target,
|
||||
fileList: input.fileList
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Errors with reading swiftc inputs
|
||||
enum SwiftcInputReaderError: Error {
|
||||
case readingFailed
|
||||
case invalidFormat
|
||||
case missingField(String)
|
||||
}
|
||||
|
||||
/// Reads SwiftC filemap that specifies all input and output files
|
||||
/// for the compilation
|
||||
protocol SwiftcInputReader {
|
||||
func read() throws -> SwiftCompilationInfo
|
||||
}
|
||||
|
||||
/// Modifies compilation info
|
||||
protocol SwiftcInputWriter {
|
||||
func write(_ info: SwiftCompilationInfo) throws
|
||||
}
|
||||
|
||||
struct SwiftCompilationInfo: Encodable, Equatable {
|
||||
var info: SwiftModuleCompilationInfo
|
||||
var files: [SwiftFileCompilationInfo]
|
||||
}
|
||||
|
||||
struct SwiftModuleCompilationInfo: Encodable, Equatable {
|
||||
// not present for incremental builds
|
||||
let dependencies: URL?
|
||||
let swiftDependencies: URL
|
||||
}
|
||||
|
||||
struct SwiftFileCompilationInfo: Encodable, Equatable {
|
||||
let file: URL
|
||||
// not present for WMO builds
|
||||
let dependencies: URL?
|
||||
// not present for 'indexbuild' builds
|
||||
let object: URL?
|
||||
// not present for WMO builds
|
||||
let swiftDependencies: URL?
|
||||
}
|
||||
|
||||
class SwiftcFilemapInputEditor: SwiftcInputReader, SwiftcInputWriter {
|
||||
|
||||
private let file: URL
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(_ file: URL, fileManager: FileManager) {
|
||||
self.file = file
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func read() throws -> SwiftCompilationInfo {
|
||||
guard let content = fileManager.contents(atPath: file.path) else {
|
||||
throw SwiftcInputReaderError.readingFailed
|
||||
}
|
||||
guard let representation = try JSONSerialization.jsonObject(with: content, options: []) as? [String: Any] else {
|
||||
throw SwiftcInputReaderError.invalidFormat
|
||||
}
|
||||
return try SwiftCompilationInfo(from: representation)
|
||||
}
|
||||
|
||||
func write(_ info: SwiftCompilationInfo) throws {
|
||||
let data = try JSONSerialization.data(withJSONObject: info.dump(), options: [.prettyPrinted])
|
||||
fileManager.createFile(atPath: file.path, contents: data, attributes: nil)
|
||||
}
|
||||
}
|
||||
|
||||
extension SwiftCompilationInfo {
|
||||
init(from object: [String: Any]) throws {
|
||||
info = try SwiftModuleCompilationInfo(from: object[""])
|
||||
files = try object.reduce([]) { prev, new in
|
||||
let (key, value) = new
|
||||
if key.isEmpty {
|
||||
return prev
|
||||
}
|
||||
let fileInfo = try SwiftFileCompilationInfo(name: key, from: value)
|
||||
return prev + [fileInfo]
|
||||
}
|
||||
}
|
||||
|
||||
func dump() -> [String: Any] {
|
||||
return files.reduce(["": info.dump()]) { prev, info in
|
||||
var result = prev
|
||||
result[info.file.path] = info.dump()
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension SwiftModuleCompilationInfo {
|
||||
init(from object: Any?) throws {
|
||||
guard let dict = object as? [String: String] else {
|
||||
throw SwiftcInputReaderError.invalidFormat
|
||||
}
|
||||
swiftDependencies = try dict.readURL(key: "swift-dependencies")
|
||||
dependencies = dict.readURL(key: "dependencies")
|
||||
}
|
||||
|
||||
func dump() -> [String: String] {
|
||||
return [
|
||||
"dependencies": dependencies?.path,
|
||||
"swift-dependencies": swiftDependencies.path,
|
||||
].compactMapValues { $0 }
|
||||
}
|
||||
}
|
||||
|
||||
extension SwiftFileCompilationInfo {
|
||||
init(name: String, from inputObject: Any) throws {
|
||||
guard let dict = inputObject as? [String: String] else {
|
||||
throw SwiftcInputReaderError.invalidFormat
|
||||
}
|
||||
file = URL(fileURLWithPath: name)
|
||||
dependencies = dict.readURL(key: "dependencies")
|
||||
object = dict.readURL(key: "object")
|
||||
swiftDependencies = dict.readURL(key: "swift-dependencies")
|
||||
}
|
||||
|
||||
func dump() -> [String: String] {
|
||||
return [
|
||||
"dependencies": dependencies?.path,
|
||||
"object": object?.path,
|
||||
"swift-dependencies": swiftDependencies?.path,
|
||||
].compactMapValues { $0 }
|
||||
}
|
||||
}
|
||||
|
||||
private extension Dictionary where Key == String, Value == String {
|
||||
func readURL(key: String) throws -> URL {
|
||||
guard let value = self[key].map(URL.init(fileURLWithPath:)) else {
|
||||
throw SwiftcInputReaderError.missingField(key)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func readURL(key: String) -> URL? {
|
||||
return self[key].map(URL.init(fileURLWithPath:))
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Performs the `swiftc` logic
|
||||
/// Depending on the mode, tries to mock the compilation (consumer)
|
||||
/// or generates and uploads artifacts (producer)
|
||||
class SwiftcOrchestrator {
|
||||
private let swiftc: SwiftcProtocol
|
||||
private let mode: SwiftcContext.SwiftcMode
|
||||
// swiftc command that should be called to generate artifacts
|
||||
private let swiftcCommand: String
|
||||
private let objcHeaderOutput: URL
|
||||
private let moduleOutput: URL
|
||||
private let arch: String
|
||||
private let artifactBuilder: ArtifactSwiftProductsBuilder
|
||||
private let shellOut: ShellOut
|
||||
private let producerFallbackCommandProcessors: [ShellCommandsProcessor]
|
||||
private let invocationStorage: InvocationStorage
|
||||
|
||||
init(
|
||||
mode: SwiftcContext.SwiftcMode,
|
||||
swiftc: SwiftcProtocol,
|
||||
swiftcCommand: String,
|
||||
objcHeaderOutput: URL,
|
||||
moduleOutput: URL,
|
||||
arch: String,
|
||||
artifactBuilder: ArtifactSwiftProductsBuilder,
|
||||
producerFallbackCommandProcessors: [ShellCommandsProcessor],
|
||||
invocationStorage: InvocationStorage,
|
||||
shellOut: ShellOut
|
||||
) {
|
||||
self.mode = mode
|
||||
self.swiftc = swiftc
|
||||
self.swiftcCommand = swiftcCommand
|
||||
self.objcHeaderOutput = objcHeaderOutput
|
||||
self.moduleOutput = moduleOutput
|
||||
self.arch = arch
|
||||
self.artifactBuilder = artifactBuilder
|
||||
self.producerFallbackCommandProcessors = producerFallbackCommandProcessors
|
||||
self.invocationStorage = invocationStorage
|
||||
self.shellOut = shellOut
|
||||
}
|
||||
|
||||
private var invocationArgs: [String] {
|
||||
let args = ProcessInfo().arguments
|
||||
// first arg is a path to the current command, drop it
|
||||
return Array(args.dropFirst())
|
||||
}
|
||||
|
||||
private func fallbackToDefault(command: String = "swiftc") {
|
||||
defaultLog("Fallbacking to compilation using \(command).")
|
||||
shellOut.switchToExternalProcess(command: command, invocationArgs: invocationArgs)
|
||||
}
|
||||
|
||||
private func fallbackToDefaultAndWait(command: String = "swiftc", args: [String]) throws {
|
||||
defaultLog("Fallbacking to compilation using \(command).")
|
||||
do {
|
||||
try shellOut.callExternalProcessAndWait(
|
||||
command: command,
|
||||
invocationArgs: Array(args.dropFirst()),
|
||||
envs: ProcessInfo.processInfo.environment
|
||||
)
|
||||
} catch ShellError.statusError(_, let exitCode) {
|
||||
exit(exitCode)
|
||||
}
|
||||
}
|
||||
|
||||
/// calls all invocations one-by-one
|
||||
/// - Parameter invocations: array or invocations: a command and all arguments
|
||||
private func callExternalInvocations(invocations: [[String]]) throws {
|
||||
try invocations.forEach { fullInvocation in
|
||||
guard let command = fullInvocation.first else {
|
||||
throw InvocationStorageError.corruptedStorage
|
||||
}
|
||||
try fallbackToDefaultAndWait(command: command, args: fullInvocation)
|
||||
}
|
||||
}
|
||||
|
||||
func run() throws {
|
||||
switch mode {
|
||||
case .consumer(.available):
|
||||
let compileStepResult = try swiftc.mockCompilation()
|
||||
do {
|
||||
if case .forceFallback = compileStepResult {
|
||||
// last-time fallback (probably a new swift file was added to the target)
|
||||
// we are responsible to call all gathered compilation steps in compilation history
|
||||
let historyCommandsToCall = try invocationStorage.retrieveAll()
|
||||
try callExternalInvocations(invocations: historyCommandsToCall)
|
||||
fallbackToDefault(command: swiftcCommand)
|
||||
} else {
|
||||
// save the current compilation invocation to the history file
|
||||
try invocationStorage.store(args: invocationArgs)
|
||||
}
|
||||
} catch {
|
||||
// The critical section is protected by a lock. Some other process already called compilation history.
|
||||
// We only need to call our current step then.
|
||||
fallbackToDefault(command: swiftcCommand)
|
||||
}
|
||||
case .consumer:
|
||||
fallbackToDefault(command: swiftcCommand)
|
||||
case .producerFast:
|
||||
let compileStepResult = try swiftc.mockCompilation()
|
||||
if case .forceFallback = compileStepResult {
|
||||
// cannot reuse cached artifact. Build it locally and upload to the server just as for the producer
|
||||
fallthrough
|
||||
}
|
||||
case .producer:
|
||||
var swiftcArgs = ProcessInfo().arguments
|
||||
swiftcArgs = try producerFallbackCommandProcessors.reduce(swiftcArgs) { args, processor in
|
||||
try processor.applyArgsRewrite(args)
|
||||
}
|
||||
try fallbackToDefaultAndWait(command: swiftcCommand, args: swiftcArgs)
|
||||
// move generated .h to the location where artifact creator expects it
|
||||
try artifactBuilder.includeObjCHeaderToTheArtifact(arch: arch, headerURL: objcHeaderOutput)
|
||||
// move generated .swiftmodule to the location where artifact creator expects it
|
||||
try artifactBuilder.includeModuleDefinitionsToTheArtifact(arch: arch, moduleURL: moduleOutput)
|
||||
|
||||
try producerFallbackCommandProcessors.forEach {
|
||||
try $0.postCommandProcessing()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Extends the swiftc product generation (when consuming cached artifact(s))
|
||||
protocol SwiftcProductGenerationPlugin {
|
||||
|
||||
/// Allows to extend the production generation
|
||||
/// - Parameter for: info of all compilation files passed to the swiftc invocation
|
||||
func generate(for: SwiftCompilationInfo) throws
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum DiskSwiftcProductsGeneratorError: Error {
|
||||
/// When a generator was asked to generate unknown swiftmodule extension file.
|
||||
/// Probably a programmer error: asking to generate excessive extensions, not listed in
|
||||
/// `SwiftmoduleFileExtension.SwiftmoduleExtensions`
|
||||
case unknownSwiftmoduleFile
|
||||
}
|
||||
|
||||
/// Generates swiftc product to the expected location
|
||||
protocol SwiftcProductsGenerator {
|
||||
/// Generates products from given files
|
||||
/// - Returns: location dir where .swiftmodule files have been placed
|
||||
func generateFrom(
|
||||
artifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
|
||||
artifactSwiftModuleObjCFile: URL
|
||||
) throws -> URL
|
||||
}
|
||||
|
||||
/// Generator that produces all products in the locations where Xcode expects it, using provided disk copier
|
||||
class DiskSwiftcProductsGenerator: SwiftcProductsGenerator {
|
||||
private let destinationSwiftmodulePaths: [SwiftmoduleFileExtension: URL]
|
||||
private let modulePathOutput: URL
|
||||
private let objcHeaderOutput: URL
|
||||
private let diskCopier: DiskCopier
|
||||
|
||||
init(
|
||||
modulePathOutput: URL,
|
||||
objcHeaderOutput: URL,
|
||||
diskCopier: DiskCopier
|
||||
) {
|
||||
self.modulePathOutput = modulePathOutput
|
||||
let modulePathBasename = modulePathOutput.deletingPathExtension()
|
||||
// all swiftmodule-related should be located next to the ".swiftmodule"
|
||||
destinationSwiftmodulePaths = Dictionary(
|
||||
uniqueKeysWithValues: SwiftmoduleFileExtension.SwiftmoduleExtensions
|
||||
.map { ext, _ in
|
||||
(ext, modulePathBasename.appendingPathExtension(ext.rawValue))
|
||||
}
|
||||
)
|
||||
self.objcHeaderOutput = objcHeaderOutput
|
||||
self.diskCopier = diskCopier
|
||||
}
|
||||
|
||||
func generateFrom(
|
||||
artifactSwiftModuleFiles sourceAtifactSwiftModuleFiles: [SwiftmoduleFileExtension: URL],
|
||||
artifactSwiftModuleObjCFile: URL
|
||||
) throws -> URL {
|
||||
// Move cached -Swift.h file to the expected location
|
||||
try diskCopier.copy(file: artifactSwiftModuleObjCFile, destination: objcHeaderOutput)
|
||||
for (ext, url) in sourceAtifactSwiftModuleFiles {
|
||||
let dest = destinationSwiftmodulePaths[ext]
|
||||
guard let destination = dest else {
|
||||
throw DiskSwiftcProductsGeneratorError.unknownSwiftmoduleFile
|
||||
}
|
||||
do {
|
||||
// Move cached .swiftmodule to the expected location
|
||||
try diskCopier.copy(file: url, destination: destination)
|
||||
} catch {
|
||||
if case .required = SwiftmoduleFileExtension.SwiftmoduleExtensions[ext] {
|
||||
throw error
|
||||
} else {
|
||||
infoLog("Optional .\(ext) file not found in the artifact at: \(destination.path)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build parent dir of the .swiftmodule file that contains a module
|
||||
return modulePathOutput.deletingLastPathComponent()
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct SwiftcArgInput {
|
||||
let objcHeaderOutput: String
|
||||
let moduleName: String
|
||||
let modulePathOutput: String
|
||||
let filemap: String
|
||||
let target: String
|
||||
let fileList: String
|
||||
|
||||
/// Manual initializer implementation required to be public
|
||||
public init(
|
||||
objcHeaderOutput: String,
|
||||
moduleName: String,
|
||||
modulePathOutput: String,
|
||||
filemap: String,
|
||||
target: String,
|
||||
fileList: String
|
||||
) {
|
||||
self.objcHeaderOutput = objcHeaderOutput
|
||||
self.moduleName = moduleName
|
||||
self.modulePathOutput = modulePathOutput
|
||||
self.filemap = filemap
|
||||
self.target = target
|
||||
self.fileList = fileList
|
||||
}
|
||||
}
|
||||
|
||||
public class XCSwiftc {
|
||||
private let command: String
|
||||
private let inputArgs: SwiftcArgInput
|
||||
private let dependenciesWriterFactory: (URL, FileManager) -> DependenciesWriter
|
||||
private let touchFactory: (URL, FileManager) -> Touch
|
||||
|
||||
public init(
|
||||
command: String,
|
||||
inputArgs: SwiftcArgInput,
|
||||
dependenciesWriter: @escaping (URL, FileManager) -> DependenciesWriter,
|
||||
touchFactory: @escaping (URL, FileManager) -> Touch
|
||||
) {
|
||||
self.command = command
|
||||
self.inputArgs = inputArgs
|
||||
dependenciesWriterFactory = dependenciesWriter
|
||||
self.touchFactory = touchFactory
|
||||
}
|
||||
|
||||
// swiftlint:disable:next function_body_length
|
||||
public func run() {
|
||||
let fileManager = FileManager.default
|
||||
let config: XCRemoteCacheConfig
|
||||
let context: SwiftcContext
|
||||
do {
|
||||
let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
|
||||
config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
|
||||
.readConfiguration()
|
||||
context = try SwiftcContext(config: config, input: inputArgs)
|
||||
} catch {
|
||||
exit(1, "FATAL: Swiftc initialization failed with error: \(error)")
|
||||
}
|
||||
let swiftcCommand = config.swiftcCommand
|
||||
let markerURL = context.tempDir.appendingPathComponent(config.modeMarkerPath)
|
||||
let markerReader = FileMarkerReader(markerURL, fileManager: fileManager)
|
||||
let markerWriter = FileMarkerWriter(markerURL, fileAccessor: fileManager)
|
||||
|
||||
let inputReader = SwiftcFilemapInputEditor(context.filemap, fileManager: fileManager)
|
||||
let fileListEditor = FileListEditor(context.fileList, fileManager: fileManager)
|
||||
let artifactOrganizer = ZipArtifactOrganizer(targetTempDir: context.tempDir, fileManager: fileManager)
|
||||
// TODO: check for allowedFile comparing a list of all inputfiles, not dependencies from a marker
|
||||
let makerReferencedFilesListScanner = FileListScannerImpl(markerReader, caseSensitive: false)
|
||||
let allowedFilesListScanner = ExceptionsFilteredFileListScanner(
|
||||
allowedFilenames: ["\(config.thinTargetMockFilename).swift"],
|
||||
disallowedFilenames: [],
|
||||
scanner: makerReferencedFilesListScanner
|
||||
)
|
||||
let artifactBuilder: ArtifactSwiftProductsBuilder = ArtifactSwiftProductsBuilderImpl(
|
||||
workingDir: context.tempDir,
|
||||
moduleName: context.moduleName,
|
||||
fileManager: fileManager
|
||||
)
|
||||
let productsGenerator = DiskSwiftcProductsGenerator(
|
||||
modulePathOutput: context.modulePathOutput,
|
||||
objcHeaderOutput: context.objcHeaderOutput,
|
||||
diskCopier: HardLinkDiskCopier(fileManager: fileManager)
|
||||
)
|
||||
let allInvocationsStorage = ExistingFileStorage(
|
||||
storageFile: context.invocationHistoryFile,
|
||||
command: swiftcCommand
|
||||
)
|
||||
// When fallbacking to local compilation do not call historical `swiftc` invocations
|
||||
// The current fallback invocation already compiles all files in a target
|
||||
let invocationStorage = FilteredInvocationStorage(
|
||||
storage: allInvocationsStorage,
|
||||
retrieveIgnoredCommands: [swiftcCommand]
|
||||
)
|
||||
let shellOut = ProcessShellOut()
|
||||
|
||||
let swiftc = Swiftc(
|
||||
inputFileListReader: fileListEditor,
|
||||
markerReader: markerReader,
|
||||
allowedFilesListScanner: allowedFilesListScanner,
|
||||
artifactOrganizer: artifactOrganizer,
|
||||
inputReader: inputReader,
|
||||
context: context,
|
||||
markerWriter: markerWriter,
|
||||
productsGenerator: productsGenerator,
|
||||
fileManager: fileManager,
|
||||
dependenciesWriterFactory: dependenciesWriterFactory,
|
||||
touchFactory: touchFactory,
|
||||
plugins: []
|
||||
)
|
||||
let orchestrator = SwiftcOrchestrator(
|
||||
mode: context.mode,
|
||||
swiftc: swiftc,
|
||||
swiftcCommand: swiftcCommand,
|
||||
objcHeaderOutput: context.objcHeaderOutput,
|
||||
moduleOutput: context.modulePathOutput,
|
||||
arch: context.arch,
|
||||
artifactBuilder: artifactBuilder,
|
||||
producerFallbackCommandProcessors: [],
|
||||
invocationStorage: invocationStorage,
|
||||
shellOut: shellOut
|
||||
)
|
||||
do {
|
||||
try orchestrator.run()
|
||||
} catch {
|
||||
exit(1, "Swiftc failed with error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
public enum Mode: String, Codable, CaseIterable {
|
||||
case consumer
|
||||
case producer
|
||||
case producerFast = "producer-fast"
|
||||
}
|
||||
@@ -1,355 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Yams
|
||||
|
||||
public enum XCRemoteCacheConfigError: Error {
|
||||
/// Obligatory configuration property is missing
|
||||
case missingConfiguration(name: String)
|
||||
}
|
||||
|
||||
public struct XCRemoteCacheConfig: Encodable {
|
||||
/// Remote cache schema version. Bump that version if RC artifact generation introduces breaking changes
|
||||
let schemaVersion = "5"
|
||||
/// Mode: consumer|producer, defaults to consumer
|
||||
var mode: Mode = .consumer
|
||||
/// Address of all remote cache replicas. The best one (with the quickest response) will be chose in xcprepare step
|
||||
/// Required to be non-empty array
|
||||
var cacheAddresses: [String] = []
|
||||
/// Address of the remote cache to use in the consumer mode
|
||||
/// If not specified, the first item in `cacheAddresses` will be used
|
||||
var recommendedCacheAddress: String = ""
|
||||
/// Probe request path to the `cacheAddresses` (relative to `cacheAddresses`)
|
||||
/// that determines the best cache to use (with the lowest latency)
|
||||
var cacheHealthPath: String = "nginx-health"
|
||||
/// Number of `cacheAddresses` probe requests
|
||||
var cacheHealthPathProbeCount: Int = 3
|
||||
/// Filepath to the file to the remote commit sha
|
||||
var remoteCommitFile: String = "build/remote-cache/arc.rc"
|
||||
/// Filepath to create xccc wrapper (that value should be equal to Xcode's CC BuildSetting)
|
||||
var xcccFile: String = "build/bin/xccc"
|
||||
/// Path, relative to $TARGET_TEMP_DIR which specifies prebuild discovery .d file
|
||||
var prebuildDiscoveryPath: String = "prebuild.d"
|
||||
/// Path, relative to $TARGET_TEMP_DIR which specifies postbuild discovery .d file
|
||||
var postbuildDiscoveryPath: String = "postbuild.d"
|
||||
/// Path, relative to $TARGET_TEMP_DIR of a maker file to enable (when exists) or disable (when missing)
|
||||
/// Remote cache mode
|
||||
/// Includes a list of all allowed input files to use remote cache
|
||||
var modeMarkerPath: String = "rc.enabled"
|
||||
/// Command for a standard C compilation (cc)
|
||||
var clangCommand: String = "clang"
|
||||
/// Command for a standard Swift compilation (swiftc)
|
||||
var swiftcCommand: String = "swiftc"
|
||||
/// Path of the primary repository that produces cache artifacts
|
||||
var primaryRepo: String = ""
|
||||
/// Main (primary) branch that produces cache artifacts (default to 'master')
|
||||
var primaryBranch: String = "master"
|
||||
/// Path to the git repo root
|
||||
var repoRoot: String = "."
|
||||
/// Number of historical commits to look for a cache artifacts
|
||||
var cacheCommitHistory: Int = 10
|
||||
/// Source root of the Xcode project
|
||||
var sourceRoot: String
|
||||
/// Fingerprint override extension (sample override `Module.swiftmodule/x86_64.swiftmodule.md5`)
|
||||
var fingerprintOverrideExtension: String = "md5"
|
||||
/// Optional configuration file that overrides project configuration
|
||||
var extraConfigurationFile: String = "user.rcinfo"
|
||||
/// Custom commit sha to publish artifact
|
||||
var publishingSha: String?
|
||||
/// Maximum age in days for artifact to be cached before being evicted
|
||||
var artifactMaximumAge: Int = 30
|
||||
/// Extra ENV keys that should be convoluted into the environment fingerprint
|
||||
var customFingerprintEnvs: [String] = []
|
||||
/// Root directory where all XCRemoteCache statistics (e.g. counters) are stored
|
||||
var statsDir: String = "~/.xccache"
|
||||
/// Number of retries for download requests
|
||||
var downloadRetries: Int = 0
|
||||
/// Number of retries for upload requests
|
||||
var uploadRetries: Int = 3
|
||||
/// Extra headers appended to all remote HTTP(S) requests
|
||||
var requestCustomHeaders: [String: String] = [:]
|
||||
/// Filename (without an extension) of the compilation input file that is used
|
||||
/// as a fake compilation for the forced-cached target (aka thin target)
|
||||
/// The filename has to be exclusive nor a suffix of any compilation file in a target
|
||||
var thinTargetMockFilename: String = "standin"
|
||||
/// A List of all targets that are not thin. If an empty array, all targets are meant to be non-thin
|
||||
/// A 'thin' target is a target-level mode that forces the cached artifact
|
||||
var focusedTargets: [String] = []
|
||||
/// Disable cache for http requests to fecth metadata and download artifacts
|
||||
var disableHttpCache: Bool = false
|
||||
/// Path, relative to $TARGET_TEMP_DIR which gathers all compilation commands that should be e
|
||||
/// xecuted if a target switches to local compilation.
|
||||
/// Example: A new `.swift` file invalidates remote arXcodeProjIntegrate.swifttifact and triggers local compilation
|
||||
/// When that happens, all previously skipped clang build steps
|
||||
/// need to be eventually called locally - this file lists all these commands.
|
||||
var compilationHistoryFile: String = "history.compile"
|
||||
/// Timeout for remote response data interval (in seconds). If an interval between data chunks is
|
||||
/// longer than a timeout, a request fails
|
||||
var timeoutResponseDataChunksInterval: Double = 20
|
||||
/// It true, any observed request timeout switches off remote cache for all targets
|
||||
var turnOffRemoteCacheOnFirstTimeout: Bool = false
|
||||
/// List of all extensions that should carry over source fingerprints. Extensions of all product files that
|
||||
/// contain non-deterministic content (absolute paths, timestamp, etc) should be included
|
||||
var productFilesExtensionsWithContentOverride = ["swiftmodule"]
|
||||
/// If true, plugins for thinning support should be enabled
|
||||
var thinningEnabled: Bool = false
|
||||
/// Module name of a target that works as a helper for thinned targets
|
||||
var thinningTargetModuleName: String = "ThinningRemoteCacheModule"
|
||||
/// Opt-in pretty json formatting for meta files
|
||||
var prettifyMetaFiles: Bool = false
|
||||
/// Secret key for AWS V4 Signature, if this is set the Authentication Header will be added
|
||||
var AWSSecretKey: String = ""
|
||||
/// Access key for AWS V4 Signature
|
||||
var AWSAccessKey: String = ""
|
||||
/// Region for AWS V4 Signature (e.g. `eu`)
|
||||
var AWSRegion: String = ""
|
||||
/// Service for AWS V4 Signature (e.g. `storage`)
|
||||
var AWSService: String = ""
|
||||
/// A dictionary of files path remapping that should be applied to make it absolute path agnostic on a list of dependencies.
|
||||
/// Useful if a project refers files out of repo root, either compilation files or precompiled dependencies.
|
||||
/// Keys represent generic replacement and values are substrings that should be replaced.
|
||||
/// Example: for mapping `["COOL_LIBRARY": "/CoolLibrary"]`
|
||||
/// `/CoolLibrary/main.swift`will be represented as `$(COOL_LIBRARY)/main.swift`).
|
||||
/// Warning: remapping order is not-deterministic so avoid remappings with multiple matchings.
|
||||
var outOfBandMappings: [String: String] = [:]
|
||||
/// If true, SSL certificate validation is disabled
|
||||
var disableCertificateVerification: Bool = false
|
||||
/// A feature flag to disable virtual file system overlay support (temporary)
|
||||
var disableVFSOverlay: Bool = false
|
||||
/// A list of extra ENVs that should be used as placeholders in the dependency list.
|
||||
/// ENV rewrite process is optimistic - does nothing if an ENV is not defined in the pre/postbuild process.
|
||||
var customRewriteEnvs: [String] = []
|
||||
}
|
||||
|
||||
extension XCRemoteCacheConfig {
|
||||
/// Merges existing config with the other config and returns a final result
|
||||
/// `other` scheme overrides existing configuration
|
||||
// swiftlint:disable:next function_body_length
|
||||
func merged(with scheme: ConfigFileScheme) -> XCRemoteCacheConfig {
|
||||
var merge = self
|
||||
merge.mode = scheme.mode ?? mode
|
||||
merge.recommendedCacheAddress = scheme.recommendedCacheAddress ?? recommendedCacheAddress
|
||||
merge.cacheAddresses = scheme.cacheAddresses ?? cacheAddresses
|
||||
merge.cacheHealthPath = scheme.cacheHealthPath ?? cacheHealthPath
|
||||
merge.cacheHealthPathProbeCount = scheme.cacheHealthPathProbeCount ?? cacheHealthPathProbeCount
|
||||
merge.remoteCommitFile = scheme.remoteCommitFile ?? remoteCommitFile
|
||||
merge.xcccFile = scheme.xcccFile ?? xcccFile
|
||||
merge.prebuildDiscoveryPath = scheme.prebuildDiscoveryPath ?? prebuildDiscoveryPath
|
||||
merge.postbuildDiscoveryPath = scheme.postbuildDiscoveryPath ?? postbuildDiscoveryPath
|
||||
merge.modeMarkerPath = scheme.modeMarkerPath ?? modeMarkerPath
|
||||
merge.clangCommand = scheme.clangCommand ?? clangCommand
|
||||
merge.swiftcCommand = scheme.swiftcCommand ?? swiftcCommand
|
||||
merge.primaryRepo = scheme.primaryRepo ?? primaryRepo
|
||||
merge.primaryBranch = scheme.primaryBranch ?? primaryBranch
|
||||
merge.repoRoot = scheme.repoRoot ?? repoRoot
|
||||
merge.cacheCommitHistory = scheme.cacheCommitHistory ?? cacheCommitHistory
|
||||
merge.fingerprintOverrideExtension = scheme.fingerprintOverrideExtension ?? fingerprintOverrideExtension
|
||||
merge.extraConfigurationFile = scheme.extraConfigurationFile ?? extraConfigurationFile
|
||||
merge.publishingSha = scheme.publishingSha ?? publishingSha
|
||||
merge.artifactMaximumAge = scheme.artifactMaximumAge ?? artifactMaximumAge
|
||||
merge.customFingerprintEnvs = scheme.customFingerprintEnvs ?? customFingerprintEnvs
|
||||
merge.statsDir = scheme.statsDir ?? statsDir
|
||||
merge.downloadRetries = scheme.downloadRetries ?? downloadRetries
|
||||
merge.uploadRetries = scheme.uploadRetries ?? uploadRetries
|
||||
merge.requestCustomHeaders = scheme.requestCustomHeaders ?? requestCustomHeaders
|
||||
merge.thinTargetMockFilename = scheme.thinTargetMockFilename ?? thinTargetMockFilename
|
||||
merge.focusedTargets = scheme.focusedTargets ?? focusedTargets
|
||||
merge.disableHttpCache = scheme.disableHttpCache ?? disableHttpCache
|
||||
merge.compilationHistoryFile = scheme.compilationHistoryFile ?? compilationHistoryFile
|
||||
merge.timeoutResponseDataChunksInterval =
|
||||
scheme.timeoutResponseDataChunksInterval ?? timeoutResponseDataChunksInterval
|
||||
merge.turnOffRemoteCacheOnFirstTimeout =
|
||||
scheme.turnOffRemoteCacheOnFirstTimeout ?? turnOffRemoteCacheOnFirstTimeout
|
||||
merge.productFilesExtensionsWithContentOverride =
|
||||
scheme.productFilesExtensionsWithContentOverride ?? productFilesExtensionsWithContentOverride
|
||||
merge.thinningEnabled = scheme.thinningEnabled ?? thinningEnabled
|
||||
merge.thinningTargetModuleName = scheme.thinningTargetModuleName ?? thinningTargetModuleName
|
||||
merge.prettifyMetaFiles = scheme.prettifyMetaFiles ?? prettifyMetaFiles
|
||||
merge.AWSAccessKey = scheme.AWSAccessKey ?? AWSAccessKey
|
||||
merge.AWSSecretKey = scheme.AWSSecretKey ?? AWSSecretKey
|
||||
merge.AWSRegion = scheme.AWSRegion ?? AWSRegion
|
||||
merge.AWSService = scheme.AWSService ?? AWSService
|
||||
merge.outOfBandMappings = scheme.outOfBandMappings ?? outOfBandMappings
|
||||
merge.disableCertificateVerification = scheme.disableCertificateVerification ?? disableCertificateVerification
|
||||
merge.disableVFSOverlay = scheme.disableVFSOverlay ?? disableVFSOverlay
|
||||
merge.customRewriteEnvs = scheme.customRewriteEnvs ?? customRewriteEnvs
|
||||
return merge
|
||||
}
|
||||
|
||||
/// Verifies all required properties and set defualts
|
||||
/// - Throws: `XCRemoteCacheConfigError` if the configuration is invalid
|
||||
/// - Returns: valid `XCRemoteCacheConfig` with configured defaults
|
||||
func verifyAndApplyDefaults() throws -> XCRemoteCacheConfig {
|
||||
var newConfig = self
|
||||
guard let fallbackCacheAddress = cacheAddresses.first else {
|
||||
throw XCRemoteCacheConfigError.missingConfiguration(name: "cache_addresses")
|
||||
}
|
||||
if recommendedCacheAddress.isEmpty {
|
||||
newConfig.recommendedCacheAddress = fallbackCacheAddress
|
||||
}
|
||||
return newConfig
|
||||
}
|
||||
}
|
||||
|
||||
/// A scheme of the user-specific overrides of configs
|
||||
struct ConfigFileScheme: Decodable {
|
||||
let mode: Mode?
|
||||
let recommendedCacheAddress: String?
|
||||
let cacheAddresses: [String]?
|
||||
let cacheHealthPath: String?
|
||||
let cacheHealthPathProbeCount: Int?
|
||||
let remoteCommitFile: String?
|
||||
let xcccFile: String?
|
||||
let prebuildDiscoveryPath: String?
|
||||
let postbuildDiscoveryPath: String?
|
||||
let modeMarkerPath: String?
|
||||
let clangCommand: String?
|
||||
let swiftcCommand: String?
|
||||
let primaryRepo: String?
|
||||
let primaryBranch: String?
|
||||
let repoRoot: String?
|
||||
let cacheCommitHistory: Int?
|
||||
let fingerprintOverrideExtension: String?
|
||||
let extraConfigurationFile: String?
|
||||
let publishingSha: String?
|
||||
let artifactMaximumAge: Int?
|
||||
let customFingerprintEnvs: [String]?
|
||||
let statsDir: String?
|
||||
let downloadRetries: Int?
|
||||
let uploadRetries: Int?
|
||||
let requestCustomHeaders: [String: String]?
|
||||
let thinTargetMockFilename: String?
|
||||
let focusedTargets: [String]?
|
||||
let disableHttpCache: Bool?
|
||||
let compilationHistoryFile: String?
|
||||
let timeoutResponseDataChunksInterval: Double?
|
||||
let turnOffRemoteCacheOnFirstTimeout: Bool?
|
||||
let productFilesExtensionsWithContentOverride: [String]?
|
||||
let thinningEnabled: Bool?
|
||||
let thinningTargetModuleName: String?
|
||||
let prettifyMetaFiles: Bool?
|
||||
let AWSSecretKey: String?
|
||||
let AWSAccessKey: String?
|
||||
let AWSRegion: String?
|
||||
let AWSService: String?
|
||||
let outOfBandMappings: [String: String]?
|
||||
let disableCertificateVerification: Bool?
|
||||
let disableVFSOverlay: Bool?
|
||||
let customRewriteEnvs: [String]?
|
||||
|
||||
// Yams library doesn't support encoding strategy, see https://github.com/jpsim/Yams/issues/84
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case mode
|
||||
case recommendedCacheAddress = "recommended_cache_address"
|
||||
case cacheAddresses = "cache_addresses"
|
||||
case cacheHealthPath = "cache_health_path"
|
||||
case cacheHealthPathProbeCount = "cache_health_path_probe_count"
|
||||
case remoteCommitFile = "remote_commit_file"
|
||||
case xcccFile = "xccc_file"
|
||||
case prebuildDiscoveryPath = "prebuild_discovery_path"
|
||||
case postbuildDiscoveryPath = "postbuild_discovery_path"
|
||||
case modeMarkerPath = "mode_marker_path"
|
||||
case clangCommand = "clang_command"
|
||||
case swiftcCommand = "swiftc_command"
|
||||
case primaryRepo = "primary_repo"
|
||||
case primaryBranch = "primary_branch"
|
||||
case repoRoot = "repo_root"
|
||||
case cacheCommitHistory = "cache_commit_history"
|
||||
case fingerprintOverrideExtension = "fingerprint_override_extension"
|
||||
case extraConfigurationFile = "extra_configuration_file"
|
||||
case publishingSha = "publishing_sha"
|
||||
case artifactMaximumAge = "artifact_maximum_age"
|
||||
case customFingerprintEnvs = "custom_fingerprint_envs"
|
||||
case statsDir = "stats_dir"
|
||||
case downloadRetries = "download_retries"
|
||||
case uploadRetries = "upload_retries"
|
||||
case requestCustomHeaders = "request_custom_headers"
|
||||
case thinTargetMockFilename = "thin_target_mock_filename"
|
||||
case focusedTargets = "focused_targets"
|
||||
case disableHttpCache = "disable_http_cache"
|
||||
case compilationHistoryFile = "compilation_history_file"
|
||||
case timeoutResponseDataChunksInterval = "timeout_response_data_chunks_interval"
|
||||
case turnOffRemoteCacheOnFirstTimeout = "turn_off_remote_cache_on_first_timeout"
|
||||
case productFilesExtensionsWithContentOverride = "product_files_extensions_with_content_override"
|
||||
case thinningEnabled = "thinning_enabled"
|
||||
case thinningTargetModuleName = "thinning_target_module_name"
|
||||
case prettifyMetaFiles = "prettify_meta_files"
|
||||
case AWSSecretKey = "aws_secret_key"
|
||||
case AWSAccessKey = "aws_access_key"
|
||||
case AWSRegion = "aws_region"
|
||||
case AWSService = "aws_service"
|
||||
case outOfBandMappings = "out_of_band_mappings"
|
||||
case disableCertificateVerification = "disable_certificate_verification"
|
||||
case disableVFSOverlay = "disable_vfs_overlay"
|
||||
case customRewriteEnvs = "custom_rewrite_envs"
|
||||
}
|
||||
}
|
||||
|
||||
enum XCRemoteCacheConfigReaderError: Error {
|
||||
case missingConfigurationFile(URL)
|
||||
case invalidConfiguration
|
||||
}
|
||||
|
||||
class XCRemoteCacheConfigReader {
|
||||
/// Name of the configuration file, required in $(SRCROOT) location
|
||||
private static let configurationFile = ".rcinfo"
|
||||
private let srcRoot: String
|
||||
private let fileManager: FileManager
|
||||
private lazy var yamlDecorer = YAMLDecoder(encoding: .utf8)
|
||||
|
||||
init(env: [String: String], fileManager: FileManager) throws {
|
||||
let explicitSrcRoot: String? = env.readEnv(key: "SRCROOT")
|
||||
srcRoot = explicitSrcRoot ?? fileManager.currentDirectoryPath
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
init(srcRootPath srcRoot: String, fileManager: FileManager) {
|
||||
self.srcRoot = srcRoot
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func readConfiguration() throws -> XCRemoteCacheConfig {
|
||||
let rootURL = URL(fileURLWithPath: srcRoot)
|
||||
let configURL = URL(fileURLWithPath: Self.configurationFile, relativeTo: rootURL)
|
||||
let userConfigs = try readUserConfig(configURL)
|
||||
var config = XCRemoteCacheConfig(sourceRoot: srcRoot).merged(with: userConfigs)
|
||||
let extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
|
||||
do {
|
||||
let extraConfig = try readUserConfig(extraConfURL)
|
||||
config = config.merged(with: extraConfig)
|
||||
} catch {
|
||||
infoLog("Extra config override failed with \(error). Skipping extra configuration")
|
||||
}
|
||||
|
||||
return try config.verifyAndApplyDefaults()
|
||||
}
|
||||
|
||||
/// Reads user configuration from a file
|
||||
private func readUserConfig(_ file: URL) throws -> ConfigFileScheme {
|
||||
let configurationContent = fileManager.contents(atPath: file.path)
|
||||
guard let configurationData = configurationContent else {
|
||||
throw XCRemoteCacheConfigReaderError.missingConfigurationFile(file)
|
||||
}
|
||||
guard let configurationString = String(data: configurationData, encoding: .utf8) else {
|
||||
throw XCRemoteCacheConfigReaderError.invalidConfiguration
|
||||
}
|
||||
return try yamlDecorer.decode(from: configurationString)
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Generates (producer) or moves (consumer) dSYM directory to include
|
||||
protocol DSYMOrganizer {
|
||||
/// Returns location of the existing dSYM directory, nil when dSYM is not required to share in the artifact
|
||||
/// In the 'producer' mode, for a non-static library that haven't already generated dSYM (Product is just "DWARF"),
|
||||
/// generates dSYM anyway to share debugging symbols with the artifact
|
||||
///
|
||||
/// - Returns: Path to the available dSYM package (generated or already existing)
|
||||
func relevantDSYMLocation() throws -> URL?
|
||||
/// Moves dSYM to the final destination, if one exists in the cached artifact
|
||||
/// - Parameter artifactPath: location of the unzipped artifact from cache
|
||||
func syncDSYM(artifactPath: URL) throws
|
||||
/// Removes all leftovers from previous dSYM synchronizations
|
||||
func cleanup() throws
|
||||
}
|
||||
|
||||
|
||||
class DynamicDSYMOrganizer: DSYMOrganizer {
|
||||
private let productURL: URL
|
||||
private let dSYMPath: URL
|
||||
private let machOType: MachOType
|
||||
private let wasDsymGenerated: Bool
|
||||
private let fileManager: FileManager
|
||||
private let shellCall: ShellCallFunction
|
||||
|
||||
init(
|
||||
productURL: URL,
|
||||
machOType: MachOType,
|
||||
dSYMPath: URL,
|
||||
wasDsymGenerated: Bool,
|
||||
fileManager: FileManager,
|
||||
shellCall: @escaping ShellCallFunction
|
||||
) {
|
||||
self.productURL = productURL
|
||||
self.machOType = machOType
|
||||
self.dSYMPath = dSYMPath
|
||||
self.wasDsymGenerated = wasDsymGenerated
|
||||
self.fileManager = fileManager
|
||||
self.shellCall = shellCall
|
||||
}
|
||||
|
||||
func relevantDSYMLocation() throws -> URL? {
|
||||
guard [.dynamicLib, .executable, .bundle].contains(machOType) else {
|
||||
return nil
|
||||
}
|
||||
guard wasDsymGenerated == false else {
|
||||
// dSYM has already been regerated
|
||||
return dSYMPath
|
||||
}
|
||||
try shellCall("dsymutil", [productURL.path, "-o", dSYMPath.path], nil, ProcessInfo.processInfo.environment)
|
||||
return dSYMPath
|
||||
}
|
||||
|
||||
|
||||
func syncDSYM(artifactPath: URL) throws {
|
||||
let dSYMFileName = dSYMPath.lastPathComponent
|
||||
let cachedDSYMPath = artifactPath.appendingPathComponent(dSYMFileName)
|
||||
if fileManager.fileExists(atPath: cachedDSYMPath.path) {
|
||||
try fileManager.spt_forceLinkItem(at: cachedDSYMPath, to: dSYMPath)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanup() throws {
|
||||
if !wasDsymGenerated && fileManager.fileExists(atPath: dSYMPath.path) {
|
||||
try fileManager.removeItem(at: dSYMPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum PhaseCacheModeControllerError: Error {
|
||||
/// Trying to disable remote cached for a target that is forced to use the cached artifact
|
||||
case cannotUseRemoteCacheForForcedCacheMode
|
||||
}
|
||||
|
||||
/// Controls Remote Cache mode:
|
||||
protocol CacheModeController {
|
||||
/// Enables remote cache for a set of allowed of input files (.swift or .m)
|
||||
/// Any compilation for a file that is not on that list should fallback to the compilation mode (if possible)
|
||||
/// or stop the build with error
|
||||
func enable(allowedInputFiles: [URL], dependencies: [URL]) throws
|
||||
/// Disables remote cache and fallbacks to source-compilation
|
||||
func disable() throws
|
||||
/// Returns true if remote cache mode is enabled
|
||||
func isEnabled() throws -> Bool
|
||||
/// Returns true if the mode controller should be disabled for that remote commit. That happens when some
|
||||
/// xcswift, xccc etc. commands disabled remote cache (e.g. new file was added to the compilation)
|
||||
func shouldDisable(for commit: RemoteCommitInfo) -> Bool
|
||||
}
|
||||
|
||||
class PhaseCacheModeController: CacheModeController {
|
||||
/// Path to the symbolic link that changes if other xcode is selected with `xcode-select -s`
|
||||
static let xcodeSelectLink: URL = URL(fileURLWithPath: "/var/db/xcode_select_link")
|
||||
private let mergeCommitFile: URL
|
||||
private let modeMarker: URL
|
||||
private let forceCached: Bool
|
||||
private let dependenciesWriter: DependenciesWriter
|
||||
private let dependenciesReader: DependenciesReader
|
||||
private let markerWriter: MarkerWriter
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(
|
||||
tempDir: URL,
|
||||
mergeCommitFile: URL,
|
||||
phaseDependencyPath: String,
|
||||
markerPath: String,
|
||||
forceCached: Bool,
|
||||
dependenciesWriter: (URL, FileManager) -> DependenciesWriter,
|
||||
dependenciesReader: (URL, FileManager) -> DependenciesReader,
|
||||
markerWriter: (URL, FileManager) -> MarkerWriter,
|
||||
fileManager: FileManager
|
||||
) {
|
||||
|
||||
self.mergeCommitFile = mergeCommitFile
|
||||
modeMarker = tempDir.appendingPathComponent(markerPath)
|
||||
self.fileManager = fileManager
|
||||
self.forceCached = forceCached
|
||||
let discoveryURL = tempDir.appendingPathComponent(phaseDependencyPath)
|
||||
self.dependenciesWriter = dependenciesWriter(discoveryURL, fileManager)
|
||||
self.dependenciesReader = dependenciesReader(discoveryURL, fileManager)
|
||||
self.markerWriter = markerWriter(modeMarker, fileManager)
|
||||
}
|
||||
|
||||
func enable(allowedInputFiles: [URL], dependencies: [URL]) throws {
|
||||
// marker file contains filepaths that contribute to the build products
|
||||
// and should invalidate all other target steps (swiftc,libtool etc.)
|
||||
let targetSensitiveFiles = dependencies + [modeMarker, Self.xcodeSelectLink]
|
||||
try markerWriter.enable(dependencies: targetSensitiveFiles)
|
||||
// All rc-phases (prebuid & postbuild) should be reenabled when new remote
|
||||
// merge commit or other Xcode is used
|
||||
let allDependencies = dependencies + [mergeCommitFile, Self.xcodeSelectLink]
|
||||
try dependenciesWriter.writeGeneric(dependencies: allDependencies)
|
||||
}
|
||||
|
||||
func disable() throws {
|
||||
guard !forceCached else {
|
||||
throw PhaseCacheModeControllerError.cannotUseRemoteCacheForForcedCacheMode
|
||||
}
|
||||
try markerWriter.disable()
|
||||
// Do not try to use remote cache anymore unless new remote cache merge commit or xcode is in use
|
||||
try dependenciesWriter.writeGeneric(dependencies: [mergeCommitFile, Self.xcodeSelectLink])
|
||||
}
|
||||
|
||||
func isEnabled() throws -> Bool {
|
||||
return fileManager.fileExists(atPath: modeMarker.path)
|
||||
}
|
||||
|
||||
/// Returns true if the phase dependency file contains a ["skipForSha": "some_sha"] entry and
|
||||
/// "some_sha" is equal to the `commit` argument
|
||||
func shouldDisable(for commit: RemoteCommitInfo) -> Bool {
|
||||
guard case .available(let commitValue) = commit else {
|
||||
return true
|
||||
}
|
||||
do {
|
||||
let rawDependencies = try dependenciesReader.readFilesAndDependencies()
|
||||
if let commitToSkip = rawDependencies[FileDependenciesWriter.skipForShaKey] {
|
||||
return commitToSkip.contains(commitValue)
|
||||
}
|
||||
} catch {
|
||||
// Gracefully don't disable a cache
|
||||
// That may happen if building a target for the first time
|
||||
errorLog("Couldn't verify if should disable RC for \(commitValue).")
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Performant DependenciesWriter manager that reuses generated dependencies file
|
||||
/// between multiple files that produce the same dependencies
|
||||
/// This class is not thread-safe
|
||||
class CachedFileDependenciesWriterFactory {
|
||||
private let dependencies: [URL]
|
||||
private let fileManager: FileManager
|
||||
private let factory: (URL, FileManager) -> DependenciesWriter
|
||||
private var templateDependencyFile: URL?
|
||||
|
||||
init(
|
||||
dependencies: [URL],
|
||||
fileManager: FileManager,
|
||||
writerFactory: @escaping (URL, FileManager) -> DependenciesWriter
|
||||
) {
|
||||
self.dependencies = dependencies
|
||||
self.fileManager = fileManager
|
||||
factory = writerFactory
|
||||
}
|
||||
|
||||
func generate(output: URL) throws {
|
||||
if let template = templateDependencyFile {
|
||||
try fileManager.spt_forceCopyItem(at: template, to: output)
|
||||
return
|
||||
}
|
||||
// Generate the template file (happens only once)
|
||||
let writer = factory(output, fileManager)
|
||||
try writer.writeGeneric(dependencies: dependencies)
|
||||
if fileManager.fileExists(atPath: output.path) {
|
||||
// the file has been correctly created
|
||||
templateDependencyFile = output
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Manages a file that collects all compilation invocations
|
||||
protocol CompilationHistoryOrganizer {
|
||||
/// Cleans a state of clang history invocations
|
||||
func reset()
|
||||
}
|
||||
|
||||
/// Manages a list of invocations stored in a file
|
||||
class CompilationHistoryFileOrganizer: CompilationHistoryOrganizer {
|
||||
private let file: URL
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(_ file: URL, fileManager: FileManager) {
|
||||
self.file = file
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func reset() {
|
||||
fileManager.createFile(atPath: file.path, contents: nil, attributes: nil)
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum DatWriterError: Error {
|
||||
/// Called when a string to dump to a file cannot be safely converted to bytes
|
||||
case invalidStringToSave(string: String)
|
||||
}
|
||||
|
||||
/// Writes step input and output files in a MachO format
|
||||
protocol DatWriter {
|
||||
func enable(dependencies: [URL], outputs: [URL]) throws
|
||||
}
|
||||
|
||||
/// Implementation of the depedency-info data file writer
|
||||
/// Mirrors clang implementation from `MachOLinkingContext::createDependencyFile`
|
||||
/// http://llvm.org/viewvc/llvm-project/lld/trunk/lib/ReaderWriter/MachO/MachOLinkingContext.cpp?view=markup
|
||||
class FileDatWriter: DatWriter {
|
||||
private static let inputFileOpcode = Data([0x10])
|
||||
private static let outputFileOpcode = Data([0x40])
|
||||
private static let separator = Data([0x0])
|
||||
|
||||
private let file: URL
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(_ file: URL, fileManager: FileManager) {
|
||||
self.file = file
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
|
||||
/// Saves input and output dependencies to the `self.file` location
|
||||
///
|
||||
/// Sample output:
|
||||
/// `{0x0}cctools-959.0.1{0x0}{0x10}inputFile1.swift{0x0}{0x10}inputFile2.m{0x0}{0x40}outputLibrary.a{0x0}`
|
||||
func enable(dependencies: [URL], outputs: [URL]) throws {
|
||||
var data = Self.separator
|
||||
try data.append("cctools-959.0.1".spt_utf8())
|
||||
data.append(Self.separator)
|
||||
|
||||
try dependencies.forEach { file in
|
||||
data.append(Self.inputFileOpcode)
|
||||
try data.append(file.path.spt_utf8())
|
||||
data.append(Self.separator)
|
||||
}
|
||||
try outputs.forEach { file in
|
||||
data.append(Self.outputFileOpcode)
|
||||
try data.append(file.path.spt_utf8())
|
||||
data.append(Self.separator)
|
||||
}
|
||||
try fileManager.spt_writeToFile(atPath: file.path, contents: data)
|
||||
}
|
||||
}
|
||||
|
||||
private extension String {
|
||||
func spt_utf8() throws -> Data {
|
||||
guard let content = data(using: .utf8) else {
|
||||
throw DatWriterError.invalidStringToSave(string: self)
|
||||
}
|
||||
return content
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum DependenciesMapping {
|
||||
/// Specifies which ENVs should be rewritten in the dependencies generation to make generic (paths agnostics)
|
||||
/// list of dependencies
|
||||
static let rewrittenEnvs = ["BUILD_DIR", "SRCROOT"]
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
import Yams
|
||||
|
||||
enum DependenciesReaderError: Error {
|
||||
case readingError
|
||||
case invalidFile
|
||||
case invalidFormat
|
||||
}
|
||||
|
||||
/// Readers for dependencies for a Make-format (.d) file
|
||||
public protocol DependenciesReader {
|
||||
/// Finds all dependencies paths
|
||||
func findDependencies() throws -> [String]
|
||||
/// Finds all files that were compiled
|
||||
func findInputs() throws -> [String]
|
||||
/// Reads raw dependency dictionary representation:
|
||||
/// * key is a filename of the dependency (or some "magicals", like Xcode's 'dependencies' or 'skipForSha')
|
||||
/// * value is an array of dependencies related with 'key' file
|
||||
func readFilesAndDependencies() throws -> [String: [String]]
|
||||
}
|
||||
|
||||
/// Parser for a single .d file
|
||||
public class FileDependenciesReader: DependenciesReader {
|
||||
private let file: URL
|
||||
private let fileManager: FileManager
|
||||
|
||||
public init(_ file: URL, accessor: FileManager) {
|
||||
self.file = file
|
||||
fileManager = accessor
|
||||
}
|
||||
|
||||
public func findDependencies() throws -> [String] {
|
||||
let yaml = try readRaw()
|
||||
|
||||
struct ParseState {
|
||||
var buffer: String = ""
|
||||
var prevChar: Character?
|
||||
var result: [String] = []
|
||||
func with(buffer: String? = nil, prevChar: Character? = nil, result: [String]? = nil) -> ParseState {
|
||||
var new = self
|
||||
new.buffer = buffer ?? new.buffer
|
||||
new.prevChar = prevChar ?? new.prevChar
|
||||
new.result = result ?? new.result
|
||||
return new
|
||||
}
|
||||
}
|
||||
|
||||
let dependencies = yaml.reduce(Set<String>()) { prev, arg1 -> Set<String> in
|
||||
let (key, value) = arg1
|
||||
switch key {
|
||||
case "dependencies":
|
||||
// 'clang' output formatting
|
||||
return Set(splitDependencyFileList(value))
|
||||
case let s where s.hasSuffix(".o") || s.hasSuffix(".bc"):
|
||||
// 'swiftc' output formatting
|
||||
// take dependencies from any .o or .bc file.
|
||||
// Note: For WMO, all .{o|bc} files have the same dependencies
|
||||
return Set(splitDependencyFileList(value))
|
||||
default:
|
||||
return prev
|
||||
}
|
||||
}
|
||||
return Array(dependencies)
|
||||
}
|
||||
|
||||
public func findInputs() throws -> [String] {
|
||||
exit(1, "TODO: implement")
|
||||
}
|
||||
|
||||
public func readFilesAndDependencies() throws -> [String: [String]] {
|
||||
let yaml = try readRaw()
|
||||
// files are space delimited
|
||||
return yaml.mapValues { $0.components(separatedBy: .whitespaces) }
|
||||
}
|
||||
|
||||
private func readRaw() throws -> [String: String] {
|
||||
guard let fileData = fileManager.contents(atPath: file.path) else {
|
||||
throw DependenciesReaderError.readingError
|
||||
}
|
||||
guard let fileString = String(data: fileData, encoding: .utf8) else {
|
||||
throw DependenciesReaderError.invalidFile
|
||||
}
|
||||
// .d matches the .yaml format
|
||||
guard let yaml = try Yams.load(yaml: fileString) as? [String: String] else {
|
||||
throw DependenciesReaderError.invalidFile
|
||||
}
|
||||
return yaml
|
||||
}
|
||||
|
||||
/// Splits space or new line separated files into a set of files
|
||||
/// It supports escaping whitespace charaters, prefixed with "\\"
|
||||
/// - Parameter string: string of whitespace charaters separated file paths
|
||||
/// - Returns: Array of all file paths
|
||||
private func splitDependencyFileList(_ string: String) -> [String] {
|
||||
struct ParseState {
|
||||
var buffer: String = ""
|
||||
var prevChar: Character?
|
||||
var result: [String] = []
|
||||
func with(buffer: String? = nil, prevChar: Character? = nil, result: [String]? = nil) -> ParseState {
|
||||
var new = self
|
||||
new.buffer = buffer ?? new.buffer
|
||||
new.prevChar = prevChar ?? new.prevChar
|
||||
new.result = result ?? new.result
|
||||
return new
|
||||
}
|
||||
}
|
||||
let parseResult = string.reduce(ParseState()) { total, char in
|
||||
switch char {
|
||||
case "\n" where total.prevChar == "\\":
|
||||
return total
|
||||
case " " where total.buffer.isEmpty:
|
||||
return total
|
||||
case " " where total.prevChar == "\\":
|
||||
return total.with(buffer: "\(total.buffer) ")
|
||||
case " ":
|
||||
return total.with(buffer: "", prevChar: nil, result: total.result + [total.buffer])
|
||||
case "\\":
|
||||
return total.with(prevChar: "\\")
|
||||
default:
|
||||
return total.with(buffer: "\(total.buffer)\(char)", prevChar: char, result: total.result)
|
||||
}
|
||||
}
|
||||
if !parseResult.buffer.isEmpty {
|
||||
return parseResult.result + [parseResult.buffer]
|
||||
}
|
||||
return parseResult.result
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Replaces paths formats between generic (placeholders-based) and local
|
||||
protocol DependenciesRemapper {
|
||||
/// Replaces all generic paths (with placeholders) to a local paths
|
||||
func replace(genericPaths: [String]) throws -> [String]
|
||||
/// Replaces all local paths to the generic dependencies paths
|
||||
func replace(localPaths: [String]) throws -> [String]
|
||||
}
|
||||
|
||||
class DependenciesRemapperComposite: DependenciesRemapper {
|
||||
private let remappers: [DependenciesRemapper]
|
||||
|
||||
init(_ remappers: [DependenciesRemapper]) {
|
||||
self.remappers = remappers
|
||||
}
|
||||
|
||||
func replace(genericPaths: [String]) throws -> [String] {
|
||||
try remappers.reversed().reduce(genericPaths) { prev, mapper in
|
||||
try mapper.replace(genericPaths: prev)
|
||||
}
|
||||
}
|
||||
|
||||
func replace(localPaths: [String]) throws -> [String] {
|
||||
try remappers.reduce(localPaths) { prev, mapper in
|
||||
try mapper.replace(localPaths: prev)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
final class StringDependenciesRemapper: DependenciesRemapper {
|
||||
struct Mapping {
|
||||
let generic: String
|
||||
let local: String
|
||||
}
|
||||
|
||||
private let mappings: [Mapping]
|
||||
|
||||
init(mappings: [Mapping]) {
|
||||
self.mappings = mappings
|
||||
}
|
||||
|
||||
func replace(genericPaths: [String]) throws -> [String] {
|
||||
return genericPaths.map { path in
|
||||
let localPath = mappings.reversed().reduce(path) { prevPath, mapping in
|
||||
prevPath.replacingOccurrences(of: mapping.generic, with: mapping.local)
|
||||
}
|
||||
return localPath
|
||||
}
|
||||
}
|
||||
|
||||
func replace(localPaths: [String]) throws -> [String] {
|
||||
return localPaths.map { path in
|
||||
let result = mappings.reduce(path) { prevPath, mapping in
|
||||
prevPath.replacingOccurrences(of: mapping.local, with: mapping.generic)
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Writer for dependencies in a Make-format (.d file)
|
||||
public protocol DependenciesWriter {
|
||||
/// Saves a list of dependencies for a set of files
|
||||
/// - Parameter dependencies: The dictionary where filepath is a key and an array it
|
||||
/// its dependencies filepath are values
|
||||
func write(dependencies: [String: [String]]) throws
|
||||
/// Saves a XCRetemoCache custom dependencies format (valid .d format) that indicates skipping that phase up,
|
||||
/// if the remote commit is equal to the provided `skipForSha`
|
||||
func write(skipForSha: String) throws
|
||||
}
|
||||
|
||||
extension DependenciesWriter {
|
||||
/// Write dependency list for a single file
|
||||
func write(file: URL, dependencies: [URL]) throws {
|
||||
try write(dependencies: [file.path: dependencies.map { $0.path }])
|
||||
}
|
||||
}
|
||||
|
||||
public class FileDependenciesWriter: DependenciesWriter {
|
||||
static let skipForShaKey = "skipForSha"
|
||||
|
||||
private let file: URL
|
||||
|
||||
public init(_ file: URL, accessor: FileManager) {
|
||||
self.file = file
|
||||
}
|
||||
|
||||
public func write(dependencies: [String: [String]]) throws {
|
||||
var content = ""
|
||||
for (file, deps) in dependencies {
|
||||
content.append(file + ": ")
|
||||
content.append(deps.joined(separator: " "))
|
||||
content.append("\n")
|
||||
}
|
||||
try content.write(to: file, atomically: true, encoding: .utf8)
|
||||
}
|
||||
|
||||
public func write(skipForSha sha: String) throws {
|
||||
try write(dependencies: [Self.skipForShaKey: [sha]])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
extension DependenciesWriter {
|
||||
func writeGeneric(dependencies: [URL]) throws {
|
||||
try write(dependencies: ["dependencies": dependencies.map { $0.path }])
|
||||
}
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Representation of a single compilation dependency
|
||||
public struct Dependency: Equatable {
|
||||
public enum Kind {
|
||||
case xcode
|
||||
case product
|
||||
case source
|
||||
case fingerprint
|
||||
case intermediate
|
||||
// Product of the target itself
|
||||
case ownProduct
|
||||
case unknown
|
||||
}
|
||||
|
||||
public let url: URL
|
||||
public let type: Kind
|
||||
|
||||
public init(url: URL, type: Kind) {
|
||||
self.url = url
|
||||
self.type = type
|
||||
}
|
||||
}
|
||||
|
||||
/// Processes raw compilation URL dependencies from .d files
|
||||
protocol DependencyProcessor {
|
||||
/// Processes a list of dependencies and provides a list of project-specific dependencies
|
||||
/// - Parameter files: raw dependency locations
|
||||
/// - Returns: array of project-specific dependencies
|
||||
func process(_ files: [URL]) -> [Dependency]
|
||||
}
|
||||
|
||||
/// Classifies raw dependencies and strips irrelevant dependencies
|
||||
class DependencyProcessorImpl: DependencyProcessor {
|
||||
private let xcodePath: String
|
||||
private let productPath: String
|
||||
private let sourcePath: String
|
||||
private let intermediatePath: String
|
||||
private let bundlePath: String?
|
||||
|
||||
init(xcode: URL, product: URL, source: URL, intermediate: URL, bundle: URL?) {
|
||||
xcodePath = xcode.path.dirPath()
|
||||
productPath = product.path.dirPath()
|
||||
sourcePath = source.path.dirPath()
|
||||
intermediatePath = intermediate.path.dirPath()
|
||||
bundlePath = bundle?.path.dirPath()
|
||||
}
|
||||
|
||||
func process(_ files: [URL]) -> [Dependency] {
|
||||
let dependencies = classify(files)
|
||||
return dependencies.filter(isRelevantDependency)
|
||||
}
|
||||
|
||||
private func classify(_ files: [URL]) -> [Dependency] {
|
||||
return files.map { file -> Dependency in
|
||||
let filePath = file.resolvingSymlinksInPath().path
|
||||
if filePath.hasPrefix(xcodePath) {
|
||||
return Dependency(url: file, type: .xcode)
|
||||
} else if filePath.hasPrefix(intermediatePath) {
|
||||
return Dependency(url: file, type: .intermediate)
|
||||
} else if let bundle = bundlePath, filePath.hasPrefix(bundle) {
|
||||
// If a target produces a bundle, explicitly classify all
|
||||
// of products to distinguish from other targets products
|
||||
return Dependency(url: file, type: .ownProduct)
|
||||
} else if filePath.hasPrefix(productPath) {
|
||||
return Dependency(url: file, type: .product)
|
||||
} else if filePath.hasPrefix(sourcePath) {
|
||||
return Dependency(url: file, type: .source)
|
||||
} else {
|
||||
return Dependency(url: file, type: .unknown)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func isRelevantDependency(_ dependency: Dependency) -> Bool {
|
||||
// Generated modulemaps may not be an actual dependency. Swift selects them as a
|
||||
// dependency because these contribute to the final module context but doesn't mean that given module has
|
||||
// been imported and it should invalidate current target when modified
|
||||
|
||||
// TODO: Recognize if the generated module was actually imported and only then it should be considered
|
||||
// as a valid Dependency
|
||||
if dependency.type == .product && dependency.url.pathExtension == "modulemap" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Skip:
|
||||
// - A fingerprint generated includes Xcode version build number so no need to analyze prepackaged Xcode files
|
||||
// - All files in `*/Interemediates/*` - this file are created on-fly for a given target
|
||||
// - Some files may depend on its own product (e.g. .m may #include *-Swift.h) - we know products will match
|
||||
// because in case of a hit, these will be taken from the artifact
|
||||
let irrelevantDependenciesType: [Dependency.Kind] = [.xcode, .intermediate, .ownProduct]
|
||||
return !irrelevantDependenciesType.contains(dependency.type)
|
||||
}
|
||||
}
|
||||
|
||||
fileprivate extension String {
|
||||
func dirPath() -> String {
|
||||
hasSuffix("/") ? self : appending("/")
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Verifies if the filename should be always disallowed/allowed. If a filename does not match with allowed/disallowed
|
||||
/// entries, the decision is handled by the underlying `scanner`
|
||||
/// Note: disallowed filenames have higher priorities than allowed ones
|
||||
class ExceptionsFilteredFileListScanner: FileListScanner {
|
||||
private let listScanner: FileListScanner
|
||||
private let allowedFilenames: [String]
|
||||
private let disallowedFilenames: [String]
|
||||
|
||||
/// Default initializer that specifies disallowed and allowed filenames (including an extention)
|
||||
/// Valid filenames: ['file.swift', 'file.m']
|
||||
/// Invalid filenames: ['somePath/file.swift', '/absolutePath/file.m']
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - allowedFilenames: a list of filenames which should always be allowed
|
||||
/// - disallowedFilenames: a list of filenames which should always be disallowed
|
||||
/// - scanner: underlying scanner that decides if non of allowed/disallowed pattern matches
|
||||
init(allowedFilenames: [String], disallowedFilenames: [String], scanner: FileListScanner) {
|
||||
self.allowedFilenames = allowedFilenames
|
||||
self.disallowedFilenames = disallowedFilenames
|
||||
listScanner = scanner
|
||||
}
|
||||
|
||||
func contains(_ url: URL) throws -> Bool {
|
||||
let filename = url.lastPathComponent
|
||||
if disallowedFilenames.contains(filename) {
|
||||
return false
|
||||
}
|
||||
if allowedFilenames.contains(filename) {
|
||||
return true
|
||||
}
|
||||
return try listScanner.contains(url)
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
protocol FileListScanner {
|
||||
/// Returns true if the url is present in the file list
|
||||
func contains(_ url: URL) throws -> Bool
|
||||
}
|
||||
|
||||
/// Finds file on a list of files provied by ListReader
|
||||
class FileListScannerImpl: FileListScanner {
|
||||
private let fileList: ListReader
|
||||
private let caseSensitive: Bool
|
||||
|
||||
init(_ fileList: ListReader, caseSensitive: Bool) {
|
||||
self.fileList = fileList
|
||||
self.caseSensitive = caseSensitive
|
||||
}
|
||||
|
||||
func contains(_ url: URL) throws -> Bool {
|
||||
if caseSensitive {
|
||||
return try fileList.listFilesURLs().contains(url)
|
||||
}
|
||||
let lowerCasePath = url.path.lowercased()
|
||||
return try fileList.listFilesURLs().lazy.contains { element in
|
||||
element.path.lowercased() == lowerCasePath
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Decides which location location should be considered in the fingerprint computation
|
||||
/// Meant to replaces machine-specific products (like .swiftmodule) with its source-aware fingerprint representation
|
||||
protocol FingerprintOverrideManager {
|
||||
/// File extensions that should be replced by an override
|
||||
var overridingFileExtensions: [String] { get }
|
||||
/// Returns a file that should be considered in the fingerprint generation
|
||||
func getFingerprintFile(_ url: Dependency) -> Dependency
|
||||
}
|
||||
|
||||
/// Manager that rewrites dependencies to the fingerprint override
|
||||
/// if the override file exists on a disk
|
||||
public class FingerprintOverrideManagerImpl: FingerprintOverrideManager {
|
||||
private let overrideExtension: String
|
||||
private let fileManager: FileManager
|
||||
let overridingFileExtensions: [String]
|
||||
|
||||
/// Initializer
|
||||
/// @param overrideExtension: all extensions that require fingerprint override
|
||||
/// @param fingerprintOverrideExtension: file extension of the fingerprint override
|
||||
/// @param fileManager: fileManager instance to check file existance
|
||||
public init(
|
||||
overridingFileExtensions: [String],
|
||||
fingerprintOverrideExtension: String,
|
||||
fileManager: FileManager
|
||||
) {
|
||||
self.overridingFileExtensions = overridingFileExtensions
|
||||
overrideExtension = fingerprintOverrideExtension
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
public func getFingerprintFile(_ dependency: Dependency) -> Dependency {
|
||||
// Require overrides only it already exists on a disk
|
||||
// If the dependency was not generated locally (e.g. distributed
|
||||
// as a binary) and misses ".{{overrideExtension}}",
|
||||
// the fingerprint of a raw file can be safely used
|
||||
let fingerprintOverrideURL = dependency.url.appendingPathExtension(overrideExtension)
|
||||
let isFileExistOnDisk = fileManager.fileExists(atPath: fingerprintOverrideURL.path)
|
||||
if overridingFileExtensions.contains(dependency.url.pathExtension) && isFileExistOnDisk {
|
||||
return Dependency(url: fingerprintOverrideURL, type: .fingerprint)
|
||||
}
|
||||
return dependency
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum FingerprintSyncerError: Error {
|
||||
case missingResourceValue(URL)
|
||||
case invalidFingerprint
|
||||
}
|
||||
|
||||
/// Syncs custom fingerprint overrides
|
||||
protocol FingerprintSyncer {
|
||||
/// Sets a fingerprint override for all files placed directly in a source location
|
||||
func decorate(sourceDir: URL, fingerprint: String) throws
|
||||
/// Deletes fingerprint overrides in the dir (if already created)
|
||||
func delete(sourceDir: URL) throws
|
||||
}
|
||||
|
||||
class FileFingerprintSyncer: FingerprintSyncer {
|
||||
/// Extension of the file that keeps fingerprint override
|
||||
private let fingerprintExtension: String
|
||||
private let dirAccessor: DirAccessor
|
||||
/// A list of all extensions that should be decorated with an override
|
||||
private let extensions: [String]
|
||||
|
||||
init(
|
||||
fingerprintOverrideExtension: String,
|
||||
dirAccessor: DirAccessor,
|
||||
extensions: [String]
|
||||
) {
|
||||
self.dirAccessor = dirAccessor
|
||||
fingerprintExtension = fingerprintOverrideExtension
|
||||
self.extensions = extensions
|
||||
}
|
||||
|
||||
func decorate(sourceDir: URL, fingerprint: String) throws {
|
||||
guard let fingerprintData = fingerprint.data(using: .utf8) else {
|
||||
throw FingerprintSyncerError.invalidFingerprint
|
||||
}
|
||||
guard case .dir = try dirAccessor.itemType(atPath: sourceDir.path) else {
|
||||
// no directory to decorate (no module was generated)
|
||||
return
|
||||
}
|
||||
let allURLs = try dirAccessor.items(at: sourceDir)
|
||||
// recursive search is not required as all files are located in a root dir
|
||||
for file in allURLs {
|
||||
if extensions.contains(file.pathExtension) {
|
||||
let fingerprintFile = file.appendingPathExtension(fingerprintExtension)
|
||||
try dirAccessor.write(toPath: fingerprintFile.path, contents: fingerprintData)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func delete(sourceDir: URL) throws {
|
||||
guard case .dir = try dirAccessor.itemType(atPath: sourceDir.path) else {
|
||||
// no directory to decorate (no module was generated)
|
||||
return
|
||||
}
|
||||
let allURLs = try dirAccessor.items(at: sourceDir)
|
||||
// recursive search is not required as all files are located in a root dir
|
||||
for file in allURLs where file.pathExtension == fingerprintExtension {
|
||||
try dirAccessor.removeItem(atPath: file.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum ListReaderError: Error {
|
||||
/// The file to read a list doesn't exist or is not readable
|
||||
case cannotReadFile
|
||||
/// The file content is invalid (e.g. cannot be represented as a String)
|
||||
case invalidContent
|
||||
}
|
||||
|
||||
/// Reads a list of files
|
||||
protocol ListReader {
|
||||
/// Fetches all dependencies
|
||||
/// - Throws: `ListReaderError`
|
||||
func listFilesURLs() throws -> [URL]
|
||||
/// Returns true if the reader is able to read a list of files
|
||||
func canRead() -> Bool
|
||||
}
|
||||
|
||||
protocol ListWriter {
|
||||
/// Writes a new list of files
|
||||
/// - Parameter list: files to save in a file list
|
||||
func writerListFilesURLs(_ list: [URL]) throws
|
||||
}
|
||||
|
||||
/// Reads&Writes files that list files using one-file-per-line format
|
||||
class FileListEditor: ListReader, ListWriter {
|
||||
private let file: URL
|
||||
private let fileManager: FileManager
|
||||
/// cached list of files
|
||||
private var cachedFiles: [URL]?
|
||||
|
||||
init(_ file: URL, fileManager: FileManager) {
|
||||
self.file = file
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func listFilesURLs() throws -> [URL] {
|
||||
if let files = cachedFiles {
|
||||
return files
|
||||
}
|
||||
guard let content = fileManager.contents(atPath: file.path) else {
|
||||
throw ListReaderError.cannotReadFile
|
||||
}
|
||||
guard let fileStrings = String(data: content, encoding: .utf8)?.split(separator: "\n") else {
|
||||
throw ListReaderError.invalidContent
|
||||
}
|
||||
let files = fileStrings.map(escapeFilename).map(URL.init(fileURLWithPath:))
|
||||
cachedFiles = files
|
||||
return files
|
||||
}
|
||||
|
||||
func canRead() -> Bool {
|
||||
return fileManager.fileExists(atPath: file.path)
|
||||
}
|
||||
|
||||
private func escapeFilename(_ path: String.SubSequence) -> String {
|
||||
String(path).replacingOccurrences(of: "\\ ", with: " ")
|
||||
}
|
||||
|
||||
private func unescapeFilename(_ path: String) -> String {
|
||||
path.replacingOccurrences(of: " ", with: "\\ ")
|
||||
}
|
||||
|
||||
func writerListFilesURLs(_ list: [URL]) throws {
|
||||
let data = list.map(\.path).map(unescapeFilename).joined(separator: "\n").data(using: .utf8)!
|
||||
fileManager.createFile(atPath: file.path, contents: data, attributes: nil)
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Reads a list of files from a marker file
|
||||
class FileMarkerReader: ListReader {
|
||||
private let file: URL
|
||||
private let fileManager: FileManager
|
||||
private var cachedFiles: [URL]?
|
||||
|
||||
init(_ file: URL, fileManager: FileManager) {
|
||||
self.file = file
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func listFilesURLs() throws -> [URL] {
|
||||
if let cachedResponse = cachedFiles {
|
||||
return cachedResponse
|
||||
}
|
||||
// Skipping first marker line `dependencies: //`
|
||||
let fileLines = try String(contentsOf: file).split(separator: "\n").dropFirst()
|
||||
let files = fileLines.map { line in
|
||||
line.replacingOccurrences(of: FileMarkerWriter.delimiter, with: "")
|
||||
}
|
||||
let filesURLs = files.map(URL.init(fileURLWithPath:))
|
||||
cachedFiles = filesURLs
|
||||
return filesURLs
|
||||
}
|
||||
|
||||
func canRead() -> Bool {
|
||||
return fileManager.fileExists(atPath: file.path)
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Manage marker file entries
|
||||
protocol MarkerWriter {
|
||||
/// Saves all dependencies
|
||||
func enable(dependencies: [URL]) throws
|
||||
/// Disables mode marker
|
||||
func disable() throws
|
||||
}
|
||||
|
||||
/// Saves a marker using a format matching .d one
|
||||
class FileMarkerWriter: MarkerWriter {
|
||||
static let delimiter = " \\"
|
||||
private let filePath: String
|
||||
private let fileAccessor: FileAccessor
|
||||
|
||||
init(_ file: URL, fileAccessor: FileAccessor) {
|
||||
filePath = file.path
|
||||
self.fileAccessor = fileAccessor
|
||||
}
|
||||
|
||||
func enable(dependencies: [URL]) throws {
|
||||
let lines = ["dependencies: "] + dependencies.map { $0.path }
|
||||
let fileContent = lines.joined(separator: "\(Self.delimiter)\n")
|
||||
try fileAccessor.write(toPath: filePath, contents: fileContent.data(using: .utf8))
|
||||
}
|
||||
|
||||
func disable() throws {
|
||||
if fileAccessor.fileExists(atPath: filePath) {
|
||||
try fileAccessor.removeItem(atPath: filePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Marker Writer that does nothing
|
||||
class NoopMarkerWriter: MarkerWriter {
|
||||
init(_ file: URL, fileManager: FileManager) {}
|
||||
|
||||
func enable(dependencies: [URL]) throws {}
|
||||
|
||||
func disable() throws {}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// File paths remapper according the virtual file system mappings
|
||||
/// - Warning: this class is not thread safe
|
||||
class OverlayDependenciesRemapper: DependenciesRemapper {
|
||||
private let overlayReader: OverlayReader
|
||||
private var mappings: [OverlayMapping]?
|
||||
|
||||
init(overlayReader: OverlayReader) {
|
||||
self.overlayReader = overlayReader
|
||||
}
|
||||
|
||||
/// Lazily Reads mappings from a file
|
||||
/// - Warning: this function is not thread safe
|
||||
private func getMappings() throws -> [OverlayMapping] {
|
||||
guard let mappings = mappings else {
|
||||
let mappings = try overlayReader.provideMappings()
|
||||
self.mappings = mappings
|
||||
return mappings
|
||||
}
|
||||
return mappings
|
||||
}
|
||||
|
||||
private func mapPath(
|
||||
_ path: String,
|
||||
source: KeyPath<OverlayMapping,URL>,
|
||||
destination: KeyPath<OverlayMapping,URL>
|
||||
) throws -> String {
|
||||
guard let mapping = try getMappings().first(where: { $0[keyPath: source].path == path }) else {
|
||||
// TODO: support partial mappings, where a directory path can be replaced with some other directory
|
||||
// no direct mapping found
|
||||
return path
|
||||
}
|
||||
return mapping[keyPath: destination].path
|
||||
}
|
||||
|
||||
func replace(genericPaths: [String]) throws -> [String] {
|
||||
try genericPaths.map {
|
||||
try mapPath($0, source: \.virtual, destination: \.local)
|
||||
}
|
||||
}
|
||||
|
||||
func replace(localPaths: [String]) throws -> [String] {
|
||||
try localPaths.map {
|
||||
try mapPath($0, source: \.local, destination: \.virtual)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Maps overlay's virtual URL with an actual (local) location
|
||||
struct OverlayMapping: Hashable {
|
||||
let virtual: URL
|
||||
let local: URL
|
||||
}
|
||||
|
||||
enum JsonOverlayReaderError: Error {
|
||||
/// The source file is missing
|
||||
case missingSourceFile(URL)
|
||||
/// The file exists but its content is invalid
|
||||
case invalidSourceContent(URL)
|
||||
/// the overlay format is not supported - either contains a nested directory or a single file
|
||||
case unsupportedFormat
|
||||
}
|
||||
/// Provides virtual file system overlay mappings
|
||||
protocol OverlayReader {
|
||||
func provideMappings() throws -> [OverlayMapping]
|
||||
}
|
||||
|
||||
class JsonOverlayReader: OverlayReader {
|
||||
|
||||
enum Mode {
|
||||
/// Interrupts the operation if the representation file is missing
|
||||
case strict
|
||||
/// Assume empty overlay mapping if the file doesn't exist
|
||||
case bestEffort
|
||||
}
|
||||
|
||||
private struct Overlay: Decodable {
|
||||
enum OverlayType: String, Decodable {
|
||||
case file
|
||||
case directory
|
||||
}
|
||||
|
||||
struct Content: Decodable {
|
||||
let externalContents: String
|
||||
let name: String
|
||||
let type: OverlayType
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case externalContents = "external-contents"
|
||||
case name
|
||||
case type
|
||||
}
|
||||
}
|
||||
|
||||
struct RootContent: Decodable {
|
||||
let contents: [Content]
|
||||
let name: String
|
||||
let type: OverlayType
|
||||
}
|
||||
let roots: [RootContent]
|
||||
}
|
||||
|
||||
private lazy var jsonDecoder = JSONDecoder()
|
||||
private let json: URL
|
||||
private let mode: Mode
|
||||
private let fileReader: FileReader
|
||||
|
||||
|
||||
init(_ json: URL, mode: Mode, fileReader: FileReader) {
|
||||
self.json = json
|
||||
self.mode = mode
|
||||
self.fileReader = fileReader
|
||||
}
|
||||
|
||||
func provideMappings() throws -> [OverlayMapping] {
|
||||
guard let jsonContent = try fileReader.contents(atPath: json.path) else {
|
||||
switch mode {
|
||||
case .strict:
|
||||
throw JsonOverlayReaderError.missingSourceFile(json)
|
||||
case .bestEffort:
|
||||
printWarning("overlay mapping file \(json) doesn't exist. Skipping overlay for the best-effort mode.")
|
||||
return []
|
||||
}
|
||||
}
|
||||
do {
|
||||
let overlay: Overlay = try jsonDecoder.decode(Overlay.self, from: jsonContent)
|
||||
let mappings: [OverlayMapping] = try overlay.roots.reduce([]) { prev, root in
|
||||
switch root.type {
|
||||
case .directory:
|
||||
//iterate all contents
|
||||
let dir = URL(fileURLWithPath: root.name)
|
||||
let mappings: [OverlayMapping] = try root.contents.map { content in
|
||||
switch content.type {
|
||||
case .file:
|
||||
let virtual = dir.appendingPathComponent(content.name)
|
||||
let local = URL(fileURLWithPath: content.externalContents)
|
||||
return .init(virtual: virtual, local: local)
|
||||
case .directory:
|
||||
throw JsonOverlayReaderError.unsupportedFormat
|
||||
}
|
||||
|
||||
}
|
||||
return prev + mappings
|
||||
case .file:
|
||||
throw JsonOverlayReaderError.unsupportedFormat
|
||||
}
|
||||
}
|
||||
return mappings
|
||||
} catch {
|
||||
switch mode {
|
||||
case .strict:
|
||||
throw error
|
||||
case .bestEffort:
|
||||
printWarning("Overlay reader has failed with an error \(error). Best-effort mode - skipping an overlay.")
|
||||
return []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
enum PathDependenciesRemapperFactoryError: Error {
|
||||
/// Remapping keys are duplicated and can lead to undetermined results
|
||||
case mappingKeyDuplication
|
||||
}
|
||||
|
||||
class PathDependenciesRemapperFactory {
|
||||
func build(
|
||||
orderKeys: [String],
|
||||
envs: [String: String],
|
||||
customMappings: [String: String]
|
||||
) throws -> StringDependenciesRemapper {
|
||||
let mappingMap = try envs.merging(customMappings) { envValue, outOfBandMapping in
|
||||
throw PathDependenciesRemapperFactoryError.mappingKeyDuplication
|
||||
}
|
||||
let mappingOrderKeys = orderKeys + customMappings.keys
|
||||
let mappings: [StringDependenciesRemapper.Mapping] = mappingOrderKeys.compactMap { key in
|
||||
guard let localURL: URL = mappingMap.readEnv(key: key) else {
|
||||
debugLog("\(key) ENV to map a dependency is not defined")
|
||||
return nil
|
||||
}
|
||||
infoLog("Found url to remapp: \(localURL). Remapping: \(localURL.standardized.path)")
|
||||
return StringDependenciesRemapper.Mapping(generic: "$(\(key))", local: localURL.standardized.path)
|
||||
}
|
||||
return StringDependenciesRemapper(mappings: mappings)
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Reads and aggregates all compilation dependencies from a single directory
|
||||
class TargetDependenciesReader: DependenciesReader {
|
||||
private let directory: URL
|
||||
private let dirScanner: DirScanner
|
||||
private let fileDependeciesReaderFactory: (URL) -> DependenciesReader
|
||||
|
||||
public init(
|
||||
_ directory: URL,
|
||||
fileDependeciesReaderFactory: @escaping (URL) -> DependenciesReader,
|
||||
dirScanner: DirScanner
|
||||
) {
|
||||
self.directory = directory
|
||||
self.dirScanner = dirScanner
|
||||
self.fileDependeciesReaderFactory = fileDependeciesReaderFactory
|
||||
}
|
||||
|
||||
// Optimized way of finding dependencies only for files that have corresponding .o file on a disk
|
||||
public func findDependencies() throws -> [String] {
|
||||
// Not calling `readFilesAndDependencies` as it may unnecessary call expensive `findDependencies()` for
|
||||
// files that eventually will not be considered
|
||||
let allURLs = try dirScanner.items(at: directory)
|
||||
let mergedDependencies = try allURLs.reduce(Set<String>()) { (prev: Set<String>, file) in
|
||||
// include only these .d files that either have corresponding .o file (incremental) or end
|
||||
// with '-master' (whole-module).
|
||||
// Otherwise .d is probably just a leftover from previous builds
|
||||
let correspondingOutputURL = file.deletingPathExtension().appendingPathExtension("o")
|
||||
let isDependencyFile = file.pathExtension == "d"
|
||||
let isWholeModuleDependencyFile = file.deletingPathExtension().lastPathComponent.hasSuffix("-master")
|
||||
// TODO: migrate to simple `lazy var` once compiling with Swift 5.4 (Xcode 12.5+)
|
||||
let correspondingFileExists = { try self.dirScanner.itemType(atPath: correspondingOutputURL.path) == .file }
|
||||
guard try isDependencyFile && (isWholeModuleDependencyFile || correspondingFileExists()) else {
|
||||
return prev
|
||||
}
|
||||
|
||||
return try prev.union(fileDependeciesReaderFactory(file).findDependencies())
|
||||
}
|
||||
return Array(mergedDependencies).sorted()
|
||||
}
|
||||
|
||||
public func findInputs() throws -> [String] {
|
||||
fatalError("TODO: implement")
|
||||
}
|
||||
|
||||
public func readFilesAndDependencies() throws -> [String: [String]] {
|
||||
let allURLs = try dirScanner.items(at: directory)
|
||||
return try allURLs.reduce([String: [String]]()) { prev, file in
|
||||
var new = prev
|
||||
new[file.path] = try fileDependeciesReaderFactory(file).findDependencies()
|
||||
return new
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Copier that physically copies files (as duplicates)
|
||||
class CopyDiskCopier: DiskCopier {
|
||||
private let fileManager: FileManager
|
||||
|
||||
init(fileManager: FileManager) {
|
||||
self.fileManager = fileManager
|
||||
}
|
||||
|
||||
func copy(file source: URL, destination: URL) throws {
|
||||
let parent = destination.deletingLastPathComponent()
|
||||
if !fileManager.fileExists(atPath: parent.path) {
|
||||
try fileManager.createDirectory(at: parent, withIntermediateDirectories: true, attributes: nil)
|
||||
}
|
||||
try fileManager.spt_forceCopyItem(at: source, to: destination)
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
// Copyright (c) 2021 Spotify AB.
|
||||
//
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Copier that moves files between two locations
|
||||
protocol DiskCopier {
|
||||
func copy(file source: URL, destination: URL) throws
|
||||
}
|
||||
|
||||
extension DiskCopier {
|
||||
/// Moves item to the directory with the same name as in the source (mimic the `cp` behaviour)
|
||||
func copy(file source: URL, directory: URL) throws {
|
||||
let fileName = source.lastPathComponent
|
||||
let destination = directory.appendingPathComponent(fileName)
|
||||
try copy(file: source, destination: destination)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user