luke
/
klayout
mirror of https://github.com/KLayout/klayout.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Trying to avoid package downloads when possible, taking the information from the index

This commit is contained in:
Matthias Koefferlein 2023-10-29 17:37:11 +01:00
parent 6dec3b0348
commit e83b3c1477
4 changed files with 128 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
src/lay/lay/laySaltDownloadManager.cc
View File

@ -27,6 +27,7 @@
#include "tlFileUtils.h"
#include "tlWebDAV.h"
#include "tlLog.h"
#include "tlEnv.h"
#include <memory>
#include <QTreeWidgetItem>
@ -38,6 +39,14 @@ namespace lay
// ----------------------------------------------------------------------------------
static bool download_package_information ()
{
// $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO
return tl::app_flag ("always-download-package-info");
}
// ----------------------------------------------------------------------------------
ConfirmationDialog::ConfirmationDialog (QWidget *parent)
: QDialog (parent), m_confirmed (false), m_cancelled (false), m_aborted (false), m_file (50000, true)
{
@ -310,43 +319,78 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt
++progress;
// Add URL and token from the package index
//
// In order to do so, we try to use the information from that package index as far as possible.
// Downloading a package definition from the original package URL may be expensive in case of
// large GIT repositories.
//
// Downloading is required if:
// - A package download is requested without a name (package can't be looked up in the package index)
// - Or a name is given, but not found in the package index
if (! p->name.empty ()) {
const lay::SaltGrain *g = salt_mine.grain_by_name (p->name);
if (! g) {
if (p->url.empty ()) {
throw tl::Exception (tl::to_string (QObject::tr ("Package '%1' not found in index - cannot resolve download URL").arg (tl::to_qstring (p->name))));
throw tl::Exception (tl::to_string (tr ("Package '%s' not found in index - cannot resolve download URL")), p->name);
}
} else {
if (p->url.empty ()) {
if (tl::verbosity() >= 20) {
tl::log << "Resolved package URL for package " << p->name << ": " << g->url ();
tl::log << tr ("Resolved package URL for package") << " '" << p->name << "': " << g->url ();
}
p->url = g->url ();
}
p->token = g->token ();
p->grain = *g;
p->downloaded = true;
}
}
try {
// @@@ Take from repo index for Git protocol?
p->grain = SaltGrain::from_url (p->url);
} catch (tl::Exception &ex) {
throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ()))));
if (! p->downloaded && download_package_information ()) {
// If requested, download package information to complete information from index or dependencies
if (tl::verbosity() >= 10) {
tl::log << tl::sprintf (tl::to_string (tr ("Reading package description for package '%s' from: '%s")), p->name, p->url);
}
try {
p->grain = SaltGrain::from_url (p->url);
p->downloaded = true;
} catch (tl::Exception &ex) {
throw tl::Exception (tl::to_string (tr ("Error fetching spec file for package from '%s': %s")), p->url, ex.msg ());
}
}
if (p->version.empty ()) {
p->version = p->grain.version ();
if (! p->downloaded) {
if (p->name.empty ()) {
throw tl::Exception (tl::to_string (tr ("No name given package from '%s' (from dependencies or command line installation request)")), p->url);
}
if (tl::verbosity() >= 10) {
tl::warn << tl::sprintf (tl::to_string (tr ("Package '%s' not downloaded from: %s. Dependencies may not be resolved.")), p->name, p->url);
}
} else {
if (p->version.empty ()) {
p->version = p->grain.version ();
}
if (p->name.empty ()) {
p->name = p->grain.name ();
}
if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) {
throw tl::Exception (tl::to_string (tr ("Package '%s': package in repository is too old (%s) to satisfy requirements (%s)")), p->name, p->grain.version (), p->version);
}
}
p->name = p->grain.name ();
p->downloaded = true;
if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) {
throw tl::Exception (tl::to_string (QObject::tr ("Package '%1': package in repository is too old (%2) to satisfy requirements (%3)").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (p->grain.version ())).arg (tl::to_qstring (p->version))));
}
}
}

4
src/lay/lay/laySaltGrain.h
View File

@ -469,6 +469,8 @@ public:
* This method will return a grain constructed from the downloaded data.
* The data is read from "URL/grain.xml". This method will throw an
* exception if an error occurs during reading.
*
* CAUTION: with GIT protocol and large repositories, this function may be very expensive.
*/
static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);
@ -477,6 +479,8 @@ public:
* The stream is a new'd object and needs to be deleted by the caller.
* "url" is the download URL on input and gets modified to match the
* actual URL if it is a relative one.
*
* CAUTION: with GIT protocol and large repositories, this function may be very expensive.
*/
static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);

12
src/lay/lay/laySaltGrainPropertiesDialog.cc
View File

@ -25,6 +25,7 @@
#include "tlString.h"
#include "tlExceptions.h"
#include "tlHttpStream.h"
#include "tlEnv.h"
#include <QFileDialog>
#include <QFileInfo>
@ -43,6 +44,14 @@ namespace lay
// ----------------------------------------------------------------------------------------------------
static bool download_package_information ()
{
// $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO
return tl::app_flag ("always-download-package-info");
}
// ----------------------------------------------------------------------------------------------------
/**
* @brief A delegate for editing a field of the dependency list
*/
@ -585,8 +594,7 @@ SaltGrainPropertiesDialog::accept ()
<< tr ("If the dependency package has a version itself, the version is automatically set to its current version.");
}
if (!d->url.empty ()) {
// @@@ TODO: only do for SVN repo
if (!d->url.empty () && download_package_information ()) {
SaltGrain gdep;
try {
gdep = SaltGrain::from_url (d->url);

93
src/lay/lay/laySaltManagerDialog.cc
View File

@ -30,6 +30,7 @@
#include "ui_SaltGrainTemplateSelectionDialog.h"
#include "tlString.h"
#include "tlExceptions.h"
#include "tlEnv.h"
#include "rba.h"
#include "pya.h"
@ -51,6 +52,14 @@ namespace lay
// --------------------------------------------------------------------------------------
static bool download_package_information ()
{
// $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO
return tl::app_flag ("always-download-package-info");
}
// --------------------------------------------------------------------------------------
/**
* @brief A tiny dialog to select a template and a name for the grain
*/
@ -1168,48 +1177,58 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex
mp_downloaded_target = details;
m_salt_mine_grain.reset (new lay::SaltGrain (*g));
// Download actual grain definition file
try {
if (download_package_information ()) {
if (g->url ().empty ()) {
throw tl::Exception (tl::to_string (tr ("No download link available")));
// Download actual grain definition file
try {
if (g->url ().empty ()) {
throw tl::Exception (tl::to_string (tr ("No download link available")));
}
QString html = tr (
"<html>"
"<body>"
"<font color=\"#c0c0c0\">"
"<h2>Fetching Package Definition ...</h2>"
"<p><b>URL</b>: %1</p>"
"</font>"
"</body>"
"</html>"
)
.arg (tl::to_qstring (g->url ()));
details->setHtml (html);
std::string url = g->url ();
m_downloaded_grain.reset (new SaltGrain ());
m_downloaded_grain->set_url (url);
// NOTE: stream_from_url may modify the URL, hence we set it again
ProcessEventCallback callback;
m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback));
m_downloaded_grain->set_url (url);
tl::InputHttpStream *http = dynamic_cast<tl::InputHttpStream *> (m_downloaded_grain_reader->base ());
if (http) {
// async reading on HTTP
http->ready ().add (this, &SaltManagerDialog::data_ready);
http->send ();
} else {
data_ready ();
}
} catch (tl::Exception &ex) {
show_error (ex);
}
QString html = tr (
"<html>"
"<body>"
"<font color=\"#c0c0c0\">"
"<h2>Fetching Package Definition ...</h2>"
"<p><b>URL</b>: %1</p>"
"</font>"
"</body>"
"</html>"
)
.arg (tl::to_qstring (g->url ()));
} else {
details->setHtml (html);
// Download denied - take information from index
m_downloaded_grain.reset (new SaltGrain (*g));
data_ready ();
std::string url = g->url ();
m_downloaded_grain.reset (new SaltGrain ());
m_downloaded_grain->set_url (url);
// NOTE: stream_from_url may modify the URL, hence we set it again
ProcessEventCallback callback;
m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback));
m_downloaded_grain->set_url (url);
tl::InputHttpStream *http = dynamic_cast<tl::InputHttpStream *> (m_downloaded_grain_reader->base ());
if (http) {
// async reading on HTTP
http->ready ().add (this, &SaltManagerDialog::data_ready);
http->send ();
} else {
data_ready ();
}
} catch (tl::Exception &ex) {
show_error (ex);
}
}