feat: setting to override source archive download command (#1815)

* Support command mocking on Windows

* Add tests

* Make archive download command configurable

* Fix validation of distribution.override

* Prevent configuring command settings locally

* Remove redundant Download function

* Add documentation

* Self-review

* Fix test on Windows when gunzip is only available from msys2

* Restructure documentation

* Add erroneously omitted user-changes.md entry for #1745

* Update user-changes.md

* Fix deprecated 'fail_on_error' in spellcheck workflow

* Switch to US English spellings

* Review

---------

Co-authored-by: Alejandro R. Mosteo <amosteo@unizar.es>

.github/workflows/spellcheck.yml

@@ -16,4 +16,4 @@ jobs:
           reporter: github-pr-annotations
           locale: "US"
           pattern: '*.md'
-          fail_on_error: true
+          fail_level: warning

doc/getting-started.md

@@ -273,6 +273,13 @@ website:
 
 * [alire.ada.dev](https://alire.ada.dev)
 
+### Using Alire with other indexes
+
+So far in this guide we have been using the community index, a central catalog
+of publicly available crates, but it is possible to host your own index for
+crates which you do not wish to make generally available. For more information,
+see [using Alire with private crates](private-crates).
+
 ## Build environment
 
 To create a build environment, `alr` sets environment variables such as

doc/private-crates.md

@@ -0,0 +1,159 @@
+# Using Alire with private crates
+
+By default, Alire uses the open-source crates available through the community
+index, but it can also be configured to fetch non-public crates (e.g. during
+development or when working with proprietary code). There are two key components
+required to make a crate available to Alire; the crate must be searchable in an
+*index* (though this requirement can be temporarily circumvented with
+[pins](catalog-format-spec#work-in-progress-dependency-overrides)), and the
+crate's source files must be fetchable from an *origin*.
+
+
+## Using a private index
+
+### Creating an index
+
+An Alire index is nothing more than a directory containing a collection of
+release manifest files in a certain directory structure. Full details of the
+format of an index can be found in
+[the catalog format specification](catalog-format-spec).
+
+To create a new index, simply create an empty directory at a location of your
+choice, and add to it a file called `index.toml` containing one line with the
+form `version = "x.x.x"`, specifying the index format used. The range of
+versions Alire is compatible with can be found by running `alr version`; when
+creating a new index you should simply use the highest version listed under
+`compatible index versions:`.
+
+If future updates to Alire affect compatibility with existing indexes, they will
+be listed in
+[BREAKING.md](https://github.com/alire-project/alire/blob/master/BREAKING.md).
+
+### Configuring an index
+
+To start using your new index, run
+```sh
+alr index --add=<path> --name=<name>
+```
+where `<name>` is a human-friendly label that `alr` will use to refer to it.
+
+It is possible to configure multiple indexes (with any conflicts resolved using
+a priority order specified by the `--before` switch). For example, you may wish
+to configure both the community index and a private index of your own
+unpublished crates.
+
+If you intend to use your private index as the only index (i.e. without also
+keeping the community index configured), you will need to add crates for the
+compiler and GPRbuild. See the community index's
+[`gnat_*` crates](https://github.com/alire-project/alire-index/tree/HEAD/index/gn) and
+[`gprbuild` crate](https://github.com/alire-project/alire-index/tree/HEAD/index/gp/gprbuild)
+for more details on how this can be achieved.
+
+### Adding a new crate to an index
+
+In order to add a crate to the index, you must create a suitable manifest file
+to describe the release, and place this manifest at the appropriate location in
+the index directory.
+
+The creation of the manifest file is automated through the `alr publish`
+command; submission to the community index can be disabled by supplying the
+`--for-private-index` switch. See [publishing](publishing) for more detail on
+this process.
+
+The resulting manifest file must then be copied to the index directory. The
+`alr publish` command will provide instructions on the correct location at which
+to place it.
+
+The newly added crate will become available for use with `alr` immediately,
+unless the crate being published contains `"provides"` definitions, in which
+case a call to `alr index --update-all` will be required.
+
+### Remote indexes
+
+You may wish to share your private index outside of your local filesystem. Any
+means of synchronizing the contents of the index directory will suffice, but
+Alire will manage this process automatically if you use a remote Git repository.
+
+It is often useful to have other files in the same repository as the index
+(a `README`, CI configuration, templates etc.), so the index itself is located
+in a first-level subdirectory of the repository (conventionally called `index/`,
+though `alr` searches for any directory containing an `index.toml` file).
+
+To start using a remote Git repository as an index, run
+```sh
+alr index --add=<URL> --name=<name>
+```
+Note that `<URL>` can point directly to the remote repository, so no local clone is required.
+
+Changes on the remote index will not take effect until `alr` performs an index
+update, either with `alr index --update-all`, or through a scheduled auto-update
+(which is performed every 24 hours by default).
+
+Once configured, there is no practical difference between the community index
+and a private index on your own remote Git repository.
+
+
+## Requiring user authentication
+
+In addition to listing crates in a private index, you may wish to limit access
+to authenticated users only. This is supported for crates and indexes made
+available as Git repositories, and for crates made available as archive files.
+
+### Git repositories
+
+Crates and indexes can be fetched as clones of a git repository, and for most
+users this option will require minimal additional configuration.
+
+Repositories are cloned and updated by calling out to the system's `git`
+command, so `git` must be configured with appropriate credentials to
+authenticate with any private origin. This usually means either Git-over-SSH
+(see your SSH client's documentation), or HTTPS with `git`'s credential handling
+(documented [here](https://git-scm.com/docs/gitcredentials)). As a general rule,
+if a `git clone` or `git fetch` on a crate/index origin succeeds, then so will
+any corresponding dependency updates with `alr`.
+
+The main consideration when using such repositories with Alire is ensuring that
+the correct form of URL is used. It must be apparent to `alr` that the URL
+refers to a Git repository (which can be disambiguated with a `git+` prefix),
+and `git` must be able to infer which protocol should be used (documented
+[here](https://git-scm.com/docs/git-clone#URLS)). It is therefore recommended
+that URLs use the schemes `git+https://` or `git+ssh://` as appropriate.
+
+Repositories on the local filesystem can be specified with the form
+`git+file:/some/path`, though this is not recommended except as a temporary
+arrangement for testing purposes.
+
+### Source archives
+
+Crates can also be fetched as an archive file (either tarball or Zip) containing
+the relevant sources. `alr` simply requires some means of downloading this file
+from a URL to a local filesystem location.
+
+The command used to fetch such files is specified by the [settings](settings)
+key `origins.archive.download_cmd`. By default `alr` uses the command
+`curl ${URL} -L -s -o ${DEST}`, which does not attempt any form of
+authentication, but this can be changed to any equivalent alternative which
+implements a desired authentication scheme. The command should download the file
+to `${DEST}` (the full file path, not a directory), and must not pollute the
+containing directory with other files.
+
+The simplest way to enable user authentication is to configure the server to
+accept HTTP Basic authentication and supply `curl` with the switch `--netrc`
+(or equivalently `-n`), which instructs it to scan the `.netrc` file in the
+user's home directory and attempt to authenticate with any credentials found
+there (see `man curl` and `man netrc` for more detail). This is achieved by
+invoking
+```sh
+alr settings --set --global origins.archive.download_cmd 'curl ${URL} -n -L -s -o ${DEST}'
+```
+Note that most terminals will perform substitutions on double-quoted strings
+containing `${SOMETHING}`, so it is important to enclose the value in
+single-quotes.
+
+If you wish to use a `.wgetrc` configuration file instead, the equivalent `wget`
+command is `'wget ${URL} -q -O ${DEST}'`.
+
+This setting only accepts a simple space-separated command, with no scripting
+functionality. If this is not sufficient, you can write more complex logic (or
+commands with arguments containing spaces) to a separate script, for instance
+by setting the value to `'python /path/to/my_script.py ${URL} ${DEST}'`.

doc/publishing.md

@@ -314,36 +314,22 @@ This will be shown as:
 ## Publishing to a local/private index
 
 Having a local or private index may be useful sometimes, be it for local
-testing, or for private crates not intended for publication.
+testing, or for private crates not intended for publication. For more
+information on private indexes, see
+[this guide](private-crates#using-a-private-index).
 
-There is no practical difference between the community index and a private index
-stored locally on disk or on your own infrastructure. An index must be located
-in a first level subdirectory of an accessible git repository or local
-filesystem location (or optionally at the top level in the case of a local
-filesystem index). This subdirectory should contain only an `index.toml`
-file and one or more `cr/crate_name` subdirectories within which the crate
-manifests themselves are located. The `index.toml` file contains one line with
-the form `version = "x.x.x"`, specifying the index format used. The range of
-versions Alire is compatible with can be found by running `alr version`, and
-breaking changes are listed in
-[BREAKING.md](https://github.com/alire-project/alire/blob/master/BREAKING.md).
-
-To start using such an index, run
-
-`alr index --add=<URL> --name=<name>`,
-
-where `<name>` is a human-friendly label that `alr` will use to refer to it.
-
-To publish a crate to a private index, run
-
-`alr publish --for-private-index [<path|URL> <commit|tag|branch>]`
-
-as described in the sections above, then place the manifest file it generates at
-the indicated path (relative to the location of `index.toml`).
-
-Additions to indexes stored locally on the disk will take effect immediately,
-unless the crate being published contains `"provides"` definitions, in which
-case an index update will be required (either with `alr index --update-all`, or
-through a scheduled auto-update) to ensure it is properly used by the dependency
-solver. An index update will always be required when publishing to a git
-repository index.
+To "publish" a crate to a private index, run
+```
+alr publish --for-private-index [<path|URL> [<commit|tag|branch>]]
+```
+where the `--for-private-index` switch disables the submission step and certain
+checks which are only applicable to the community index, and the remaining
+arguments function as described above. This will generate a manifest file which
+you can place at the indicated path (relative to the location of `index.toml`)
+in your private index.
+
+One important thing to note is that publishing from a local repository will
+detect the URL configured as the Git remote (as displayed by
+`git remote show origin`). If this is not configured with the recommended form
+(as discussed [here](private-crates#git-repositories)), you may wish to pass the
+desired URL explicitly.

doc/user-changes.md

@@ -6,9 +6,28 @@ stay on top of `alr` new features.
 
 ## Release `2.1`
 
+### Custom download command for archive crates
+
+PR [#1815](https://github.com/alire-project/alire/pull/1815)
+
+The command used to download a crate as a source archive can now be configured
+using the `origins.archive.download_cmd` key of `alr settings`, instead of using
+a hard-coded `curl` command. The tokens `${URL}` and `${DEST}` are replaced by
+the origin URL and destination file path respectively.
+
+For example
+```sh
+alr settings --set --global origins.archive.download_cmd 'curl ${URL} --netrc -L -s -o ${DEST}'
+```
+configures `alr` to use the default command with the addition of the switch
+`--netrc` (which instructs `curl` to use the login information in the `.netrc`
+file found in the user's home directory).
+
+The default behavior is unchanged.
+
 ### Abbreviated `--tree` output for repeating dependencies
 
-PR [1814](https://github.com/alire-project/alire/pull/1814)
+PR [#1814](https://github.com/alire-project/alire/pull/1814)
 
 By default, repeated dependencies are now omitted by `--tree` output, e.g.:
 
@@ -40,7 +59,7 @@ switch.
 
 ### Faster `alr search` without resolving dependencies
 
-PR [1799](https://github.com/alire-project/alire/pull/1799)
+PR [#1799](https://github.com/alire-project/alire/pull/1799)
 
 `alr search` no longer solves dependencies of releases by default, in order to
 speed up the command. The `--solve` switch can be used to achieve the old
@@ -51,19 +70,29 @@ In the new default situation, releases that have dependencies are marked with a
 dependencies and replace the '?' with either nothing for a solvable release or
 the usual 'X' if dependencies are unsatisfiable.
 
+### Support for private indexes with `alr publish --for-private-index`
+
+PR [#1745](https://github.com/alire-project/alire/pull/1745)
+
+Automated manifest generation with `alr publish` can now be performed for crates
+which are not intended for submission to the community index by supplying the
+`--for-private-index` switch. This has the same effects as `--skip-submit`, and
+additionally disables a number of checks that enforce submission requirements
+specific to the community index.
+
 ## Release `2.0`
 
 ### `ALIRE_SETTINGS_DIR` replaces `ALR_CONFIG`
 
-PR [1625](https://github.com/alire-project/alire/pull/1625)
+PR [#1625](https://github.com/alire-project/alire/pull/1625)
 
 This reflects the new nomenclature of Alire settings versus crate
 configuration. Also, it better reflects that the effect is on the whole library
 and not only the `alr` command-line tool.
 
 ### `alr settings` replaces `alr config`
 
-PR [1617](https://github.com/alire-project/alire/pull/1617)
+PR [#1617](https://github.com/alire-project/alire/pull/1617)
 
 The `alr settings` command replaces the `alr config` command. This change is
 introduced to tackle the confusion between the configuration of the Alire

src/alire/alire-environment-loading.adb

@@ -1,5 +1,5 @@
 with Alire_Early_Elaboration;
-with Alire.Environment.Formatting;
+with Alire.Formatting;
 with Alire.GPR;
 with Alire.Platforms.Current;
 with Alire.Properties.Scenarios;
@@ -125,7 +125,7 @@ package body Alire.Environment.Loading is
                          Formatting.Format
                            (Act.Value,
                             Formatting.For_Manifest_Environment (Release_Base),
-                            Is_Path => True);
+                            Convert_Path_Seps => True);
             begin
                case Act.Action is
 

src/alire/alire-environment-formatting.adb → src/alire/alire-formatting.adb

@@ -5,7 +5,7 @@ with Ada.Strings.Unbounded; use Ada.Strings.Unbounded;
 with Alire.OS_Lib;
 with Alire.Platforms.Current;
 
-package body Alire.Environment.Formatting is
+package body Alire.Formatting is
 
    --------------
    -- Contains --
@@ -22,6 +22,22 @@ package body Alire.Environment.Formatting is
    function Value (This : Replacements; Pattern : Patterns) return String
    is (This (Pattern));
 
+   --------------------------
+   -- For_Archive_Download --
+   --------------------------
+
+   function For_Archive_Download (URL : String;
+                                  Destination : Absolute_Path)
+                                  return Replacements
+   is
+      Result : Replacements;
+   begin
+      Result.Insert (Formatting.URL, URL);
+      Result.Insert (Formatting.Dest, Destination);
+
+      return Result;
+   end For_Archive_Download;
+
    ------------------------------
    -- For_Manifest_Environment --
    ------------------------------
@@ -98,9 +114,9 @@ package body Alire.Environment.Formatting is
    -- Format --
    ------------
 
-   function Format (Item    : String;
-                    Repl    : Replacements;
-                    Is_Path : Boolean)
+   function Format (Item              : String;
+                    Repl              : Replacements;
+                    Convert_Path_Seps : Boolean)
                     return String
    is
       -------------
@@ -121,7 +137,7 @@ package body Alire.Environment.Formatting is
             Replace_Slice (Str, From, To, "TEST");
 
          else
-            raise Unknown_Formatting_Key;
+            raise Unknown_Formatting_Key with "Unknown formatting key: " & Id;
          end if;
       end Replace;
 
@@ -170,11 +186,11 @@ package body Alire.Environment.Formatting is
 
       --  For final usage, we use the native separator
 
-      if Is_Path then
+      if Convert_Path_Seps then
          return To_Native (+Result);
       else
          return +Result;
       end if;
    end Format;
 
-end Alire.Environment.Formatting;
+end Alire.Formatting;