From e83b3c1477601d61d1c2fdf455aef1e70e3ccfae Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 29 Oct 2023 17:37:11 +0100 Subject: [PATCH] Trying to avoid package downloads when possible, taking the information from the index --- src/lay/lay/laySaltDownloadManager.cc | 72 ++++++++++++---- src/lay/lay/laySaltGrain.h | 4 + src/lay/lay/laySaltGrainPropertiesDialog.cc | 12 ++- src/lay/lay/laySaltManagerDialog.cc | 93 +++++++++++++-------- 4 files changed, 128 insertions(+), 53 deletions(-) diff --git a/src/lay/lay/laySaltDownloadManager.cc b/src/lay/lay/laySaltDownloadManager.cc index cc0b625d7..ea0202810 100644 --- a/src/lay/lay/laySaltDownloadManager.cc +++ b/src/lay/lay/laySaltDownloadManager.cc @@ -27,6 +27,7 @@ #include "tlFileUtils.h" #include "tlWebDAV.h" #include "tlLog.h" +#include "tlEnv.h" #include #include @@ -38,6 +39,14 @@ namespace lay // ---------------------------------------------------------------------------------- +static bool download_package_information () +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info"); +} + +// ---------------------------------------------------------------------------------- + ConfirmationDialog::ConfirmationDialog (QWidget *parent) : QDialog (parent), m_confirmed (false), m_cancelled (false), m_aborted (false), m_file (50000, true) { @@ -310,43 +319,78 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt ++progress; // Add URL and token from the package index + // + // In order to do so, we try to use the information from that package index as far as possible. + // Downloading a package definition from the original package URL may be expensive in case of + // large GIT repositories. + // + // Downloading is required if: + // - A package download is requested without a name (package can't be looked up in the package index) + // - Or a name is given, but not found in the package index + if (! p->name.empty ()) { const lay::SaltGrain *g = salt_mine.grain_by_name (p->name); if (! g) { if (p->url.empty ()) { - throw tl::Exception (tl::to_string (QObject::tr ("Package '%1' not found in index - cannot resolve download URL").arg (tl::to_qstring (p->name)))); + throw tl::Exception (tl::to_string (tr ("Package '%s' not found in index - cannot resolve download URL")), p->name); } } else { if (p->url.empty ()) { if (tl::verbosity() >= 20) { - tl::log << "Resolved package URL for package " << p->name << ": " << g->url (); + tl::log << tr ("Resolved package URL for package") << " '" << p->name << "': " << g->url (); } p->url = g->url (); } p->token = g->token (); + p->grain = *g; + p->downloaded = true; } } - try { - // @@@ Take from repo index for Git protocol? - p->grain = SaltGrain::from_url (p->url); - } catch (tl::Exception &ex) { - throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ())))); + if (! p->downloaded && download_package_information ()) { + + // If requested, download package information to complete information from index or dependencies + if (tl::verbosity() >= 10) { + tl::log << tl::sprintf (tl::to_string (tr ("Reading package description for package '%s' from: '%s")), p->name, p->url); + } + try { + p->grain = SaltGrain::from_url (p->url); + p->downloaded = true; + } catch (tl::Exception &ex) { + throw tl::Exception (tl::to_string (tr ("Error fetching spec file for package from '%s': %s")), p->url, ex.msg ()); + } + } - if (p->version.empty ()) { - p->version = p->grain.version (); + if (! p->downloaded) { + + if (p->name.empty ()) { + throw tl::Exception (tl::to_string (tr ("No name given package from '%s' (from dependencies or command line installation request)")), p->url); + } + + if (tl::verbosity() >= 10) { + tl::warn << tl::sprintf (tl::to_string (tr ("Package '%s' not downloaded from: %s. Dependencies may not be resolved.")), p->name, p->url); + } + + } else { + + if (p->version.empty ()) { + p->version = p->grain.version (); + } + if (p->name.empty ()) { + p->name = p->grain.name (); + } + + if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) { + throw tl::Exception (tl::to_string (tr ("Package '%s': package in repository is too old (%s) to satisfy requirements (%s)")), p->name, p->grain.version (), p->version); + } + } - p->name = p->grain.name (); p->downloaded = true; - if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) { - throw tl::Exception (tl::to_string (QObject::tr ("Package '%1': package in repository is too old (%2) to satisfy requirements (%3)").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (p->grain.version ())).arg (tl::to_qstring (p->version)))); - } - } } diff --git a/src/lay/lay/laySaltGrain.h b/src/lay/lay/laySaltGrain.h index 58e429ffa..bf0f2ef90 100644 --- a/src/lay/lay/laySaltGrain.h +++ b/src/lay/lay/laySaltGrain.h @@ -469,6 +469,8 @@ public: * This method will return a grain constructed from the downloaded data. * The data is read from "URL/grain.xml". This method will throw an * exception if an error occurs during reading. + * + * CAUTION: with GIT protocol and large repositories, this function may be very expensive. */ static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); @@ -477,6 +479,8 @@ public: * The stream is a new'd object and needs to be deleted by the caller. * "url" is the download URL on input and gets modified to match the * actual URL if it is a relative one. + * + * CAUTION: with GIT protocol and large repositories, this function may be very expensive. */ static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); diff --git a/src/lay/lay/laySaltGrainPropertiesDialog.cc b/src/lay/lay/laySaltGrainPropertiesDialog.cc index fa6321f6e..22ad35005 100644 --- a/src/lay/lay/laySaltGrainPropertiesDialog.cc +++ b/src/lay/lay/laySaltGrainPropertiesDialog.cc @@ -25,6 +25,7 @@ #include "tlString.h" #include "tlExceptions.h" #include "tlHttpStream.h" +#include "tlEnv.h" #include #include @@ -43,6 +44,14 @@ namespace lay // ---------------------------------------------------------------------------------------------------- +static bool download_package_information () +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info"); +} + +// ---------------------------------------------------------------------------------------------------- + /** * @brief A delegate for editing a field of the dependency list */ @@ -585,8 +594,7 @@ SaltGrainPropertiesDialog::accept () << tr ("If the dependency package has a version itself, the version is automatically set to its current version."); } - if (!d->url.empty ()) { - // @@@ TODO: only do for SVN repo + if (!d->url.empty () && download_package_information ()) { SaltGrain gdep; try { gdep = SaltGrain::from_url (d->url); diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index 98f084ab7..de2531de8 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -30,6 +30,7 @@ #include "ui_SaltGrainTemplateSelectionDialog.h" #include "tlString.h" #include "tlExceptions.h" +#include "tlEnv.h" #include "rba.h" #include "pya.h" @@ -51,6 +52,14 @@ namespace lay // -------------------------------------------------------------------------------------- +static bool download_package_information () +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info"); +} + +// -------------------------------------------------------------------------------------- + /** * @brief A tiny dialog to select a template and a name for the grain */ @@ -1168,48 +1177,58 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex mp_downloaded_target = details; m_salt_mine_grain.reset (new lay::SaltGrain (*g)); - // Download actual grain definition file - try { + if (download_package_information ()) { - if (g->url ().empty ()) { - throw tl::Exception (tl::to_string (tr ("No download link available"))); + // Download actual grain definition file + try { + + if (g->url ().empty ()) { + throw tl::Exception (tl::to_string (tr ("No download link available"))); + } + + QString html = tr ( + "" + "" + "" + "

Fetching Package Definition ...

" + "

URL: %1

" + "" + "" + "" + ) + .arg (tl::to_qstring (g->url ())); + + details->setHtml (html); + + std::string url = g->url (); + + m_downloaded_grain.reset (new SaltGrain ()); + m_downloaded_grain->set_url (url); + + // NOTE: stream_from_url may modify the URL, hence we set it again + ProcessEventCallback callback; + m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); + m_downloaded_grain->set_url (url); + + tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); + if (http) { + // async reading on HTTP + http->ready ().add (this, &SaltManagerDialog::data_ready); + http->send (); + } else { + data_ready (); + } + + } catch (tl::Exception &ex) { + show_error (ex); } - QString html = tr ( - "" - "" - "" - "

Fetching Package Definition ...

" - "

URL: %1

" - "" - "" - "" - ) - .arg (tl::to_qstring (g->url ())); + } else { - details->setHtml (html); + // Download denied - take information from index + m_downloaded_grain.reset (new SaltGrain (*g)); + data_ready (); - std::string url = g->url (); - - m_downloaded_grain.reset (new SaltGrain ()); - m_downloaded_grain->set_url (url); - - // NOTE: stream_from_url may modify the URL, hence we set it again - ProcessEventCallback callback; - m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); - m_downloaded_grain->set_url (url); - - tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); - if (http) { - // async reading on HTTP - http->ready ().add (this, &SaltManagerDialog::data_ready); - http->send (); - } else { - data_ready (); - } - - } catch (tl::Exception &ex) { - show_error (ex); } }