From 40bdd63ee47f7db071a774408e3a96554cf41b32 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 25 Oct 2023 00:15:52 +0200 Subject: [PATCH 01/26] First integration of libgit2 --- build.sh | 9 + src/klayout.pri | 9 + src/lay/lay/layApplication.cc | 2 +- src/lay/lay/layInit.cc | 2 +- src/lay/lay/laySalt.cc | 25 ++- src/lay/lay/laySaltController.cc | 33 +++- src/lay/lay/laySaltDownloadManager.cc | 29 ++- src/lay/lay/laySaltDownloadManager.h | 10 +- src/lay/lay/laySaltGrain.cc | 91 ++++++--- src/lay/lay/laySaltGrain.h | 53 +++++- src/lay/lay/laySaltGrainPropertiesDialog.cc | 2 +- src/lay/lay/laySaltManagerDialog.cc | 31 ++- src/tl/tl/tl.pro | 12 +- src/tl/tl/tlGit.cc | 198 ++++++++++++++++++++ src/tl/tl/tlGit.h | 108 +++++++++++ src/tl/unit_tests/tlGitTests.cc | 59 ++++++ src/tl/unit_tests/unit_tests.pro | 1 + 17 files changed, 624 insertions(+), 50 deletions(-) create mode 100644 src/tl/tl/tlGit.cc create mode 100644 src/tl/tl/tlGit.h create mode 100644 src/tl/unit_tests/tlGitTests.cc diff --git a/build.sh b/build.sh index 8d647deed..a2b7c9280 100755 --- a/build.sh +++ b/build.sh @@ -39,6 +39,7 @@ HAVE_QT=1 HAVE_PNG=0 HAVE_CURL=0 HAVE_EXPAT=0 +HAVE_GIT2=1 RUBYINCLUDE="" RUBYINCLUDE2="" @@ -209,6 +210,9 @@ while [ "$*" != "" ]; do -libexpat) HAVE_EXPAT=1 ;; + -nolibgit2) + HAVE_GIT2=0 + ;; -qt5) echo "*** WARNING: -qt5 option is ignored - Qt version is auto-detected now." ;; @@ -495,6 +499,9 @@ fi if [ $HAVE_PNG != 0 ]; then echo " Uses libpng for PNG generation" fi +if [ $HAVE_GIT2 != 0 ]; then + echo " Uses libgit2 for Git access" +fi if [ "$RPATH" = "" ]; then RPATH="$BIN" fi @@ -578,6 +585,7 @@ echo " HAVE_64BIT_COORD=$HAVE_64BIT_COORD" echo " HAVE_CURL=$HAVE_CURL" echo " HAVE_PNG=$HAVE_PNG" echo " HAVE_EXPAT=$HAVE_EXPAT" +echo " HAVE_GIT2=$HAVE_GIT2" echo " RPATH=$RPATH" mkdir -p $BUILD @@ -650,6 +658,7 @@ qmake_options=( HAVE_CURL="$HAVE_CURL" HAVE_EXPAT="$HAVE_EXPAT" HAVE_PNG="$HAVE_PNG" + HAVE_GIT2="$HAVE_GIT2" PREFIX="$BIN" RPATH="$RPATH" KLAYOUT_VERSION="$KLAYOUT_VERSION" diff --git a/src/klayout.pri b/src/klayout.pri index 0cb25c0ab..27a7051a0 100644 --- a/src/klayout.pri +++ b/src/klayout.pri @@ -103,6 +103,15 @@ equals(HAVE_PTHREADS, "1") { DEFINES += HAVE_PTHREADS } +equals(HAVE_GIT2, "1") { + !isEmpty(BITS_PATH) { + include($$BITS_PATH/git2/git2.pri) + } else { + LIBS += -lgit2 + } + DEFINES += HAVE_GIT2 +} + equals(HAVE_RUBY, "1") { !isEmpty(BITS_PATH) { include($$BITS_PATH/ruby/ruby.pri) diff --git a/src/lay/lay/layApplication.cc b/src/lay/lay/layApplication.cc index e7c54f768..1b008fb7a 100644 --- a/src/lay/lay/layApplication.cc +++ b/src/lay/lay/layApplication.cc @@ -770,7 +770,7 @@ ApplicationBase::init_app () size_t local_folders = (lay::get_appdata_path ().empty () ? 0 : 1); for (std::vector ::const_iterator p = m_klayout_path.begin (); p != m_klayout_path.end (); ++p) { - if (p - m_klayout_path.begin () < local_folders) { + if (size_t (p - m_klayout_path.begin ()) < local_folders) { mc->add_path (*p, tl::to_string (QObject::tr ("Local")), std::string (), false); } else if (m_klayout_path.size () == 1 + local_folders) { mc->add_path (*p, tl::to_string (QObject::tr ("Global")), std::string (), true); diff --git a/src/lay/lay/layInit.cc b/src/lay/lay/layInit.cc index 18e423c3a..ec108ab79 100644 --- a/src/lay/lay/layInit.cc +++ b/src/lay/lay/layInit.cc @@ -152,7 +152,7 @@ void init (const std::vector &_paths) try { s_plugins.push_back (do_load_plugin (imp)); modules.insert (*im); - } catch (tl::Exception (&ex)) { + } catch (tl::Exception &ex) { tl::error << ex.msg (); } } diff --git a/src/lay/lay/laySalt.cc b/src/lay/lay/laySalt.cc index d6fcd501f..4f5f428b7 100644 --- a/src/lay/lay/laySalt.cc +++ b/src/lay/lay/laySalt.cc @@ -21,11 +21,16 @@ */ #include "laySalt.h" + #include "tlString.h" #include "tlFileUtils.h" #include "tlLog.h" #include "tlInternational.h" #include "tlWebDAV.h" +#if defined(HAVE_GIT2) +# include "tlGit.h" +#endif + #include "lymMacro.h" #include @@ -485,9 +490,23 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, double timeout, t if (templ.url ().find ("http:") == 0 || templ.url ().find ("https:") == 0) { - // otherwise download from the URL - tl::info << QObject::tr ("Downloading package from '%1' to '%2' ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); - res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback); + // otherwise download from the URL using Git or SVN + + if (templ.protocol () == Git) { + +#if defined(HAVE_GIT2) + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); + res = tl::GitObject::download (templ.url (), target.path (), templ.branch (), timeout, callback); +#else + throw tl::Exception (tl::to_string (QObject::tr ("Unable to install package '%1' - git protocol not compiled in").arg (tl::to_qstring (target.name ())))); +#endif + + } else if (templ.protocol () == WebDAV || templ.protocol () == DefaultProtocol) { + + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); + res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback); + + } } else { diff --git a/src/lay/lay/laySaltController.cc b/src/lay/lay/laySaltController.cc index 7c99e30db..2a10a8894 100644 --- a/src/lay/lay/laySaltController.cc +++ b/src/lay/lay/laySaltController.cc @@ -203,11 +203,40 @@ SaltController::install_packages (const std::vector &packages, bool } if (n.find ("http:") == 0 || n.find ("https:") == 0 || n.find ("file:") == 0 || n[0] == '/' || n[0] == '\\') { + // its a URL - manager.register_download (std::string (), std::string (), n, v); + manager.register_download (std::string (), std::string (), n, DefaultProtocol, std::string (), v); + + } else if (n.find ("git@") == 0) { + + // git protocol: + // "git@" + // "git@[]" + + std::string url (n, 4); + size_t br = url.find ("["); + std::string branch; + if (br != std::string::npos && url.back () == ']') { + branch = std::string (url, br + 1, url.size () - br - 2); + url = std::string (url, 0, br); + } + + manager.register_download (std::string (), std::string (), url, Git, branch, v); + + } else if (n.find ("svn@") == 0) { + + // svn protocol: + // "svn@" + + std::string url (n, 4); + // its a URL + manager.register_download (std::string (), std::string (), url, WebDAV, std::string (), v); + } else { + // its a plain name - manager.register_download (n, std::string (), std::string (), v); + manager.register_download (n, std::string (), std::string (), DefaultProtocol, std::string (), v); + } } diff --git a/src/lay/lay/laySaltDownloadManager.cc b/src/lay/lay/laySaltDownloadManager.cc index e104e3531..4d9032f6c 100644 --- a/src/lay/lay/laySaltDownloadManager.cc +++ b/src/lay/lay/laySaltDownloadManager.cc @@ -58,7 +58,7 @@ ConfirmationDialog::ConfirmationDialog (QWidget *parent) } void -ConfirmationDialog::add_info (const std::string &name, bool update, const std::string &version, const std::string &url) +ConfirmationDialog::add_info (const std::string &name, bool update, const std::string &version, const std::string &url, Protocol protocol, const std::string &branch) { QTreeWidgetItem *item = new QTreeWidgetItem (list); m_items_by_name.insert (std::make_pair (name, item)); @@ -68,7 +68,18 @@ ConfirmationDialog::add_info (const std::string &name, bool update, const std::s item->setText (0, tl::to_qstring (name)); item->setText (1, update ? tr ("UPDATE") : tr ("INSTALL")); item->setText (2, tl::to_qstring (version)); - item->setText (3, tl::to_qstring (url)); + + if (protocol == WebDAV) { + item->setText (3, tl::to_qstring ("svn@" + url)); + } else if (protocol == Git) { + if (branch.empty ()) { + item->setText (3, tl::to_qstring ("git@" + url)); + } else { + item->setText (3, tl::to_qstring ("git@" + url + "[" + branch + "]")); + } + } else { + item->setText (3, tl::to_qstring (url)); + } for (int column = 0; column < list->colorCount (); ++column) { item->setData (column, Qt::ForegroundRole, QVariant (QBrush (update ? QColor (Qt::blue) : QColor (Qt::black)))); @@ -169,9 +180,9 @@ SaltDownloadManager::SaltDownloadManager () } void -SaltDownloadManager::register_download (const std::string &name, const std::string &token, const std::string &url, const std::string &version) +SaltDownloadManager::register_download (const std::string &name, const std::string &token, const std::string &url, Protocol protocol, const std::string &branch, const std::string &version) { - m_registry.push_back (Descriptor (name, token, url, version)); + m_registry.push_back (Descriptor (name, token, url, protocol, branch, version)); } void @@ -244,7 +255,7 @@ SaltDownloadManager::compute_list (const lay::Salt &salt, const lay::Salt &salt_ if (tl::verbosity() >= 20) { tl::log << "Considering for update as dependency: " << d->name << " (" << d->version << ") with URL " << d->url; } - m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->version)); + m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->protocol, d->branch, d->version)); } else { if (tl::verbosity() >= 20) { @@ -257,7 +268,7 @@ SaltDownloadManager::compute_list (const lay::Salt &salt, const lay::Salt &salt_ if (tl::verbosity() >= 20) { tl::log << "Considering for download as dependency: " << d->name << " (" << d->version << ") with URL " << d->url; } - m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->version)); + m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->protocol, d->branch, d->version)); } @@ -330,7 +341,7 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt } try { - p->grain = SaltGrain::from_url (p->url); + p->grain = SaltGrain::from_url (p->url, p->protocol, p->branch); } catch (tl::Exception &ex) { throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ())))); } @@ -387,7 +398,7 @@ SaltDownloadManager::make_confirmation_dialog (QWidget *parent, const lay::Salt const lay::SaltGrain *g = salt.grain_by_name (p->name); if (g) { // \342\206\222 is UTF-8 "right arrow" - dialog->add_info (p->name, true, g->version () + " \342\206\222 " + p->version, p->url); + dialog->add_info (p->name, true, g->version () + " \342\206\222 " + p->version, p->url, p->protocol, p->branch); } } @@ -395,7 +406,7 @@ SaltDownloadManager::make_confirmation_dialog (QWidget *parent, const lay::Salt for (std::vector::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) { const lay::SaltGrain *g = salt.grain_by_name (p->name); if (!g) { - dialog->add_info (p->name, false, p->version, p->url); + dialog->add_info (p->name, false, p->version, p->url, p->protocol, p->branch); } } diff --git a/src/lay/lay/laySaltDownloadManager.h b/src/lay/lay/laySaltDownloadManager.h index a81e0f973..04c3ef2b5 100644 --- a/src/lay/lay/laySaltDownloadManager.h +++ b/src/lay/lay/laySaltDownloadManager.h @@ -53,7 +53,7 @@ Q_OBJECT public: ConfirmationDialog (QWidget *parent); - void add_info (const std::string &name, bool update, const std::string &version, const std::string &url); + void add_info (const std::string &name, bool update, const std::string &version, const std::string &url, Protocol protocol, const std::string &branch); bool is_confirmed () const { return m_confirmed; } bool is_cancelled () const { return m_cancelled; } @@ -108,7 +108,7 @@ public: * * The target directory can be empty. In this case, the downloader will pick an appropriate one. */ - void register_download (const std::string &name, const std::string &token, const std::string &url, const std::string &version); + void register_download (const std::string &name, const std::string &token, const std::string &url, Protocol protocol, const std::string &branch, const std::string &version); /** * @brief Computes the dependencies after all required packages have been registered @@ -145,8 +145,8 @@ public: private: struct Descriptor { - Descriptor (const std::string &_name, const std::string &_token, const std::string &_url, const std::string &_version) - : name (_name), token (_token), url (_url), version (_version), downloaded (false) + Descriptor (const std::string &_name, const std::string &_token, const std::string &_url, Protocol _protocol, const std::string &_branch, const std::string &_version) + : name (_name), token (_token), url (_url), protocol (_protocol), branch (_branch), version (_version), downloaded (false) { } bool operator< (const Descriptor &other) const @@ -170,6 +170,8 @@ private: std::string name; std::string token; std::string url; + Protocol protocol; + std::string branch; std::string version; bool downloaded; lay::SaltGrain grain; diff --git a/src/lay/lay/laySaltGrain.cc b/src/lay/lay/laySaltGrain.cc index 932ceb310..b956e11a0 100644 --- a/src/lay/lay/laySaltGrain.cc +++ b/src/lay/lay/laySaltGrain.cc @@ -25,8 +25,11 @@ #include "tlString.h" #include "tlXMLParser.h" #include "tlHttpStream.h" -#include "tlWebDAV.h" #include "tlFileUtils.h" +#include "tlWebDAV.h" +#if defined(HAVE_GIT2) +# include "tlGit.h" +#endif #include #include @@ -41,7 +44,7 @@ namespace lay static const std::string grain_filename = "grain.xml"; SaltGrain::SaltGrain () - : m_hidden (false) + : m_protocol (DefaultProtocol), m_hidden (false) { // .. nothing yet .. } @@ -65,7 +68,10 @@ SaltGrain::operator== (const SaltGrain &other) const m_license == other.m_license && m_hidden == other.m_hidden && m_authored_time == other.m_authored_time && - m_installed_time == other.m_installed_time; + m_installed_time == other.m_installed_time && + m_protocol == other.m_protocol && + m_branch == other.m_branch + ; } void @@ -110,6 +116,18 @@ SaltGrain::set_url (const std::string &u) m_url = u; } +void +SaltGrain::set_branch (const std::string &b) +{ + m_branch = b; +} + +void +SaltGrain::set_protocol (const Protocol &p) +{ + m_protocol = p; +} + void SaltGrain::set_title (const std::string &t) { @@ -253,18 +271,10 @@ SaltGrain::compare_versions (const std::string &v1, const std::string &v2) } } -std::string -SaltGrain::spec_url (const std::string &url) +const std::string & +SaltGrain::spec_file () { - std::string res = url; - if (! res.empty()) { - // TODO: use system path separator unless this is a URL - if (res [res.size () - 1] != '/') { - res += "/"; - } - res += grain_filename; - } - return res; + return grain_filename; } bool @@ -400,6 +410,30 @@ struct ImageConverter } }; +struct ProtocolConverter +{ + std::string to_string (const Protocol &protocol) const + { + if (protocol == lay::WebDAV) { + return std::string ("svn"); + } else if (protocol == lay::Git) { + return std::string ("git"); + } else { + return std::string (); + } + } + + void from_string (const std::string &s, Protocol &res) const + { + res = lay::DefaultProtocol; + if (s == "svn" || s == "SVN" || s == "WebDAV") { + res = lay::WebDAV; + } else if (s == "git" || s == "Git" || s == "GIT") { + res = lay::Git; + } + } +}; + static tl::XMLElementList *sp_xml_elements = 0; tl::XMLElementList & @@ -416,6 +450,8 @@ SaltGrain::xml_elements () tl::make_member (&SaltGrain::doc, &SaltGrain::set_doc, "doc") + tl::make_member (&SaltGrain::doc_url, &SaltGrain::set_doc_url, "doc-url") + tl::make_member (&SaltGrain::url, &SaltGrain::set_url, "url") + + tl::make_member (&SaltGrain::branch, &SaltGrain::set_branch, "branch") + + tl::make_member (&SaltGrain::protocol, &SaltGrain::set_protocol, "protocol", ProtocolConverter ()) + tl::make_member (&SaltGrain::license, &SaltGrain::set_license, "license") + tl::make_member (&SaltGrain::author, &SaltGrain::set_author, "author") + tl::make_member (&SaltGrain::author_contact, &SaltGrain::set_author_contact, "author-contact") + @@ -426,6 +462,8 @@ SaltGrain::xml_elements () tl::make_element (&SaltGrain::begin_dependencies, &SaltGrain::end_dependencies, &SaltGrain::add_dependency, "depends", tl::make_member (&SaltGrainDependency::name, "name") + tl::make_member (&SaltGrainDependency::url, "url") + + tl::make_member (&SaltGrainDependency::protocol, "protocol", ProtocolConverter ()) + + tl::make_member (&SaltGrainDependency::branch, "branch") + tl::make_member (&SaltGrainDependency::version, "version") ) ); @@ -512,7 +550,7 @@ SaltGrain::from_path (const std::string &path) } tl::InputStream * -SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStreamCallback *callback) +SaltGrain::stream_from_url (std::string &url, Protocol protocol, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { if (url.empty ()) { throw tl::Exception (tl::to_string (QObject::tr ("No download link available"))); @@ -533,19 +571,30 @@ SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStrea } - std::string spec_url = SaltGrain::spec_url (url); - if (spec_url.find ("http:") == 0 || spec_url.find ("https:") == 0) { - return tl::WebDAVObject::download_item (spec_url, timeout, callback); + if (url.find ("http:") == 0 || url.find ("https:") == 0) { + + if (protocol == lay::Git) { +#if defined(HAVE_GIT2) + return tl::GitObject::download_item (url, SaltGrain::spec_file (), branch, timeout, callback); +#else + throw tl::Exception (tl::to_string (QObject::tr ("Cannot download from Git - Git support not compiled in"))); +#endif + } else { + return tl::WebDAVObject::download_item (url + "/" + SaltGrain::spec_file (), timeout, callback); + } + } else { - return new tl::InputStream (spec_url); + + return new tl::InputStream (url); + } } SaltGrain -SaltGrain::from_url (const std::string &url_in, double timeout, tl::InputHttpStreamCallback *callback) +SaltGrain::from_url (const std::string &url_in, Protocol protocol, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { std::string url = url_in; - std::unique_ptr stream (stream_from_url (url, timeout, callback)); + std::unique_ptr stream (stream_from_url (url, protocol, branch, timeout, callback)); SaltGrain g; g.load (*stream); diff --git a/src/lay/lay/laySaltGrain.h b/src/lay/lay/laySaltGrain.h index 92ef33e19..1919e98ab 100644 --- a/src/lay/lay/laySaltGrain.h +++ b/src/lay/lay/laySaltGrain.h @@ -39,6 +39,15 @@ namespace tl namespace lay { +/** + * @brief An enum describing the protocol to use for download + */ +enum Protocol { + DefaultProtocol = 0, + WebDAV = 1, + Git = 2 +}; + /** * @brief A descriptor for one dependency * A dependency can be specified either through a name (see name property) @@ -49,8 +58,14 @@ namespace lay */ struct SaltGrainDependency { + SaltGrainDependency () + : protocol (DefaultProtocol) + { } + std::string name; std::string url; + Protocol protocol; + std::string branch; std::string version; bool operator== (const SaltGrainDependency &other) const @@ -341,6 +356,32 @@ public: */ void set_url (const std::string &u); + /** + * @brief Gets the download protocol + */ + const Protocol &protocol () const + { + return m_protocol; + } + + /** + * @brief Sets the download protocol + */ + void set_protocol (const Protocol &p); + + /** + * @brief Gets the Git branch + */ + const std::string &branch () const + { + return m_branch; + } + + /** + * @brief Sets the Git branch + */ + void set_branch (const std::string &b); + /** * @brief Gets a value indicating whether the grain is hidden * A grain can be hidden (in Salt.Mine) if it's a pure dependency package @@ -466,21 +507,23 @@ public: * This method will return a grain constructed from the downloaded data. * The data is read from "URL/grain.xml". This method will throw an * exception if an error occurs during reading. + * protocol is the protocol to use and branch the Git branch. */ - static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static SaltGrain from_url (const std::string &url, Protocol protocol, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** * @brief Returns a stream prepared for downloading the grain * The stream is a new'd object and needs to be deleted by the caller. * "url" is the download URL on input and gets modified to match the * actual URL if it is a relative one. + * protocol is the protocol to use and branch the Git branch. */ - static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static tl::InputStream *stream_from_url (std::string &url, Protocol protocol, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** - * @brief Forms the spec file download URL from a given download URL + * @brief Gets the name of the spec file ("grain.xml") */ - static std::string spec_url (const std::string &url); + static const std::string &spec_file (); /** * @brief Returns a value indicating whether the given path represents is a grain @@ -499,6 +542,8 @@ private: std::string m_author; std::string m_author_contact; std::string m_license; + Protocol m_protocol; + std::string m_branch; bool m_hidden; QDateTime m_authored_time, m_installed_time; QImage m_icon, m_screenshot; diff --git a/src/lay/lay/laySaltGrainPropertiesDialog.cc b/src/lay/lay/laySaltGrainPropertiesDialog.cc index 7a3f098b9..200bce4eb 100644 --- a/src/lay/lay/laySaltGrainPropertiesDialog.cc +++ b/src/lay/lay/laySaltGrainPropertiesDialog.cc @@ -586,7 +586,7 @@ SaltGrainPropertiesDialog::accept () if (!d->url.empty ()) { SaltGrain gdep; try { - gdep = SaltGrain::from_url (d->url); + gdep = SaltGrain::from_url (d->url, d->protocol, d->branch); if (gdep.name () != d->name) { dependencies_alert->error () << tr ("Package name obtained from download URL is not the expected name.") << tl::endl << tr ("Downloaded name: ") << gdep.name () << tl::endl diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index f439be7b8..4e135e266 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -35,6 +35,7 @@ #include "pya.h" #include +#include #include #include #include @@ -682,7 +683,7 @@ BEGIN_PROTECTED SaltGrain *g = model->grain_from_index (index); // NOTE: checking for valid_name prevents bad entries inside the download list if (g && model->is_marked (g->name ()) && SaltGrain::valid_name (g->name ())) { - manager.register_download (g->name (), g->token (), g->url (), g->version ()); + manager.register_download (g->name (), g->token (), g->url (), g->protocol (), g->branch (), g->version ()); any = true; } } @@ -1133,6 +1134,24 @@ BEGIN_PROTECTED END_PROTECTED } +namespace +{ + +/** + * @brief A callback to keep the UI alive (mainly used for Git grain retrieval) + */ +class ProcessEventCallback + : public tl::InputHttpStreamCallback +{ +public: + virtual void wait_for_input () + { + QApplication::processEvents (QEventLoop::ExcludeUserInputEvents); + } +}; + +} + void SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTextWidget *details) { @@ -1166,16 +1185,22 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex "" "" ) - .arg (tl::to_qstring (SaltGrain::spec_url (g->url ()))); + .arg (tl::to_qstring (g->url ())); details->setHtml (html); std::string url = g->url (); + Protocol protocol = g->protocol (); + std::string branch = g->branch (); + m_downloaded_grain.reset (new SaltGrain ()); m_downloaded_grain->set_url (url); + m_downloaded_grain->set_protocol (protocol); + m_downloaded_grain->set_branch (branch); // NOTE: stream_from_url may modify the URL, hence we set it again - m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url)); + ProcessEventCallback callback; + m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, protocol, branch, 60.0, &callback)); m_downloaded_grain->set_url (url); tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 558134601..5a9f73562 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -121,7 +121,17 @@ HEADERS = \ tlUniqueName.h \ tlRecipe.h \ tlSelect.h \ - tlEnv.h + tlEnv.h + +equals(HAVE_GIT2, "1") { + + HEADERS += \ + tlGit.h + + SOURCES += \ + tlGit.cc + +} equals(HAVE_CURL, "1") { diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc new file mode 100644 index 000000000..59e619234 --- /dev/null +++ b/src/tl/tl/tlGit.cc @@ -0,0 +1,198 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlGit.h" +#include "tlFileUtils.h" +#include "tlProgress.h" +#include "tlStaticObjects.h" + +#include +#include + +namespace tl +{ + +// --------------------------------------------------------------- +// Library initialization helper + +namespace { + +class InitHelper +{ +public: + InitHelper () + { + git_libgit2_init (); + } + + ~InitHelper () + { + git_libgit2_shutdown (); + } + + static void ensure_initialized () + { + if (ms_init) { + return; + } + + ms_init = new InitHelper (); + tl::StaticObjects::reg (&ms_init); + } + +private: + static InitHelper *ms_init; +}; + +InitHelper *InitHelper::ms_init = 0; + +} + +// --------------------------------------------------------------- +// GitCollection implementation + +GitObject::GitObject (const std::string &local_path) + : m_local_path (local_path), m_is_temp (false) +{ + InitHelper::ensure_initialized (); + + if (local_path.empty ()) { + // @@@ generic tempnam on Windows/Posix ... + char tmp[] = "git2klayoutXXXXXX"; + mkstemp (tmp); + m_local_path = tmp; + m_is_temp = true; + } + + tl::mkpath (m_local_path); + + // ensures the directory is clean + tl::rm_dir_recursive (m_local_path); // @@@ TODO: error handling? + tl::mkpath (m_local_path); +} + +GitObject::~GitObject () +{ + if (m_is_temp) { + tl::rm_dir_recursive (m_local_path); + } +} + +static int +fetch_progress (const git_indexer_progress *stats, void *payload) +{ + tl::RelativeProgress *progress = reinterpret_cast (payload); + + // first half of progress + size_t count = size_t (5000.0 * double (stats->received_objects) / double (std::max (1u, stats->total_objects)) + 1e-10); + progress->set (count); + + return 0; +} + +static void +checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload) +{ + tl::RelativeProgress *progress = reinterpret_cast (payload); + + // first half of progress + size_t count = size_t (5000.0 * double (cur) / double (std::max (size_t (1), tot)) + 1e-10); + progress->set (count + 5000u); +} + + +void +GitObject::read (const std::string &url, const std::string &filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + // @@@ use callback, timeout? + tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/); + + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + + const char *paths_cstr[1]; + paths_cstr[0] = filter.c_str (); + if (! filter.empty ()) { + checkout_opts.paths.count = 1; + checkout_opts.paths.strings = (char **) &paths_cstr; + } + + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE | + GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; + checkout_opts.progress_cb = &checkout_progress; + checkout_opts.progress_payload = (void *) &progress; + + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + + clone_opts.checkout_opts = checkout_opts; + + // NOTE: really has to be a branch! Tags won't work. + if (! branch.empty ()) { + clone_opts.checkout_branch = branch.c_str (); + } + +#if LIBGIT2_VER_MAJOR > 1 || (LIBGIT2_VER_MAJOR == 1 && LIBGIT2_VER_MINOR >= 7) + clone_opts.fetch_opts.depth = 1; // shallow (single commit) +#endif + clone_opts.fetch_opts.callbacks.transfer_progress = &fetch_progress; + clone_opts.fetch_opts.callbacks.payload = (void *) &progress; + + // Do the clone + git_repository *cloned_repo = NULL; + int error = git_clone (&cloned_repo, url.c_str (), m_local_path.c_str (), &clone_opts); + if (error != 0) { + const git_error *err = git_error_last (); + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo (%d): %s")), err->klass, (const char *) err->message); + } + + if (cloned_repo) { + git_repository_free (cloned_repo); + // remove the worktree we don't need + tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + } +} + +bool +GitObject::download (const std::string &url, const std::string &target, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + GitObject obj (target); + obj.read (url, std::string (), branch, timeout, callback); + return false; +} + +tl::InputStream * +GitObject::download_item (const std::string &url, const std::string &file, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + GitObject obj; + obj.read (url, file, branch, timeout, callback); + + // extract the file and return a memory blob, so we can delete the temp folder + + tl::InputStream file_stream (tl::combine_path (obj.local_path (), file)); + std::string data = file_stream.read_all (); + + char *data_copy = new char [data.size ()]; + memcpy (data_copy, data.c_str (), data.size ()); + return new tl::InputStream (new tl::InputMemoryStream (data_copy, data.size (), true)); +} + +} diff --git a/src/tl/tl/tlGit.h b/src/tl/tl/tlGit.h new file mode 100644 index 000000000..cbc11bda9 --- /dev/null +++ b/src/tl/tl/tlGit.h @@ -0,0 +1,108 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlGit +#define HDR_tlGit + +#include "tlCommon.h" +#include "tlStream.h" + +#include +#include + +#if !defined(HAVE_GIT2) +# error "tlGit.h can only be used with libgit2 enabled" +#endif + +namespace tl +{ + +class InputHttpStreamCallback; + +/** + * @brief Represents an object from a Git URL + * This object can be a file or collection + */ +class TL_PUBLIC GitObject +{ +public: + /** + * @brief Open a stream with the given URL + * + * The local_path is the path where to store the files. + * If empty, a temporary folder is created and destroyed once the "GitObject" goes out of scope. + */ + GitObject (const std::string &local_path = std::string ()); + + /** + * @brief Destructor + */ + ~GitObject (); + + /** + * @brief Populates the collection from the given URL + * + * "filter" can be a top-level file to download. If filter is non-empty, + * sparse mode is chosen. + */ + void read (const std::string &url, const std::string &filter, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + + /** + * @brief Downloads the collection or file with the given URL + * + * This method will download the Git object from url to the file path + * given in "target". + * + * For file download, the target must be the path of the target file. + * For collection download, the target must be a directory path. In this + * case, the target directory must exist already. + * + * Sub-directories are created if required. + * + * This method throws an exception if the directory structure could + * not be obtained or downloading of one file failed. + */ + static bool download (const std::string &url, const std::string &target, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + + /** + * @brief Gets a stream object for downloading the single item of the given URL + * + * The file needs to be a top-level object. + * The stream object returned needs to be deleted by the caller. + */ + static tl::InputStream *download_item (const std::string &url, const std::string &file, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + +private: + std::string m_local_path; + bool m_is_temp; + + const std::string &local_path () const + { + return m_local_path; + } +}; + +} + +#endif + diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc new file mode 100644 index 000000000..e7e3c1544 --- /dev/null +++ b/src/tl/unit_tests/tlGitTests.cc @@ -0,0 +1,59 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if defined(HAVE_GIT2) + +#include "tlGit.h" +#include "tlUnitTest.h" + +// @@@ +static std::string test_url ("https://github.com/klayoutmatthias/xsection.git"); + +TEST(1) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string (), std::string ()); + + printf("@@@ done: %s\n", path.c_str ()); +} + +TEST(2) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("LICENSE"), std::string ()); + + printf("@@@ done: %s\n", path.c_str ()); +} + +TEST(3) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string (), std::string ("brxxx")); + + printf("@@@ done: %s\n", path.c_str ()); +} + +#endif + diff --git a/src/tl/unit_tests/unit_tests.pro b/src/tl/unit_tests/unit_tests.pro index c648dcb1b..2e0bab909 100644 --- a/src/tl/unit_tests/unit_tests.pro +++ b/src/tl/unit_tests/unit_tests.pro @@ -20,6 +20,7 @@ SOURCES = \ tlExpressionTests.cc \ tlFileSystemWatcherTests.cc \ tlFileUtilsTests.cc \ + tlGitTests.cc \ tlHttpStreamTests.cc \ tlIncludeTests.cc \ tlInt128SupportTests.cc \ From 17fd5e9238b82ce27c49dd2ca91f551a03f5672c Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 25 Oct 2023 21:57:48 +0200 Subject: [PATCH 02/26] Basic Git client implemented. --- src/tl/tl/tlFileUtils.cc | 42 ++++++++++++ src/tl/tl/tlFileUtils.h | 9 ++- src/tl/tl/tlGit.cc | 65 ++++++++++++++++-- src/tl/unit_tests/tlFileUtilsTests.cc | 59 ++++++++++++++++ src/tl/unit_tests/tlGitTests.cc | 96 ++++++++++++++++++++++++--- 5 files changed, 255 insertions(+), 16 deletions(-) diff --git a/src/tl/tl/tlFileUtils.cc b/src/tl/tl/tlFileUtils.cc index e575dbb63..747d21c6f 100644 --- a/src/tl/tl/tlFileUtils.cc +++ b/src/tl/tl/tlFileUtils.cc @@ -584,6 +584,48 @@ cp_dir_recursive (const std::string &source, const std::string &target) return true; } +bool +mv_dir_recursive (const std::string &source, const std::string &target) +{ + std::vector entries; + std::string path = tl::absolute_file_path (source); + std::string path_to = tl::absolute_file_path (target); + + bool error = false; + + entries = dir_entries (path, false /*without_files*/, true /*with_dirs*/); + for (std::vector::const_iterator e = entries.begin (); e != entries.end (); ++e) { + std::string tc = tl::combine_path (path_to, *e); + if (! mkpath (tc)) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to create target directory: ") << tc; +#endif + error = true; + } else if (! mv_dir_recursive (tl::combine_path (path, *e), tc)) { + error = true; + } + } + + entries = dir_entries (path, true /*with_files*/, false /*without_dirs*/); + for (std::vector::const_iterator e = entries.begin (); e != entries.end (); ++e) { + if (! tl::rename_file (tl::combine_path (path, *e), tl::combine_path (path_to, *e))) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to move file from ") << tl::combine_path (path, *e) << tr (" to ") << tl::combine_path (path_to, *e); +#endif + error = true; + } + } + + if (! tl::rm_dir (path)) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to remove folder ") << path; +#endif + error = true; + } + + return ! error; +} + std::string absolute_path (const std::string &s) { std::vector parts = split_path (absolute_file_path (s)); diff --git a/src/tl/tl/tlFileUtils.h b/src/tl/tl/tlFileUtils.h index f192599ff..0649b4078 100644 --- a/src/tl/tl/tlFileUtils.h +++ b/src/tl/tl/tlFileUtils.h @@ -52,11 +52,18 @@ bool TL_PUBLIC rm_dir_recursive (const std::string &path); bool TL_PUBLIC mkpath (const std::string &path); /** - * @brief Recursively remove the given directory, the files from that directory and all sub-directories (version with std::string) + * @brief Recursively copy the given directory * @return True, if successful. false otherwise. */ bool TL_PUBLIC cp_dir_recursive (const std::string &source, const std::string &target); +/** + * @brief Recursively move the contents of the given directory + * @return True, if successful. false otherwise. + * After this operation, the source directory is deleted. + */ +bool TL_PUBLIC mv_dir_recursive (const std::string &source, const std::string &target); + /** * @brief Gets the absolute path for a given file path * This will deliver the directory of the file as absolute path. diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 59e619234..bb0182640 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -122,8 +122,28 @@ checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload) void -GitObject::read (const std::string &url, const std::string &filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { + std::string url = org_url; + std::string filter = org_filter; + + std::string subdir; + + std::string url_terminator (".git/"); + size_t url_end = url.find (url_terminator); + if (url_end != std::string::npos) { + + subdir = std::string (url, url_end + url_terminator.size ()); + + url = std::string (url, 0, url_end + url_terminator.size () - 1); + if (filter.empty ()) { + filter = subdir + "/**"; + } else { + filter = subdir + "/" + filter; + } + + } + // @@@ use callback, timeout? tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/); @@ -136,8 +156,10 @@ GitObject::read (const std::string &url, const std::string &filter, const std::s checkout_opts.paths.strings = (char **) &paths_cstr; } + /* checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; + @@@*/ checkout_opts.progress_cb = &checkout_progress; checkout_opts.progress_payload = (void *) &progress; @@ -161,13 +183,44 @@ GitObject::read (const std::string &url, const std::string &filter, const std::s int error = git_clone (&cloned_repo, url.c_str (), m_local_path.c_str (), &clone_opts); if (error != 0) { const git_error *err = git_error_last (); - throw tl::Exception (tl::to_string (tr ("Error cloning Git repo (%d): %s")), err->klass, (const char *) err->message); + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo: %s")), (const char *) err->message); } - if (cloned_repo) { - git_repository_free (cloned_repo); - // remove the worktree we don't need - tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + if (! cloned_repo) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - no data available"))); + } + + git_repository_free (cloned_repo); + + // remove the worktree as we don't need it + tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + + // pull subfolder files to target path level + if (! subdir.empty ()) { + + std::string pp = tl::combine_path (m_local_path, subdir); + if (! tl::is_dir (pp)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to fetch subdirectory: ")) + pp); + } + + // rename the source to a temporary folder so we don't overwrite the source folder with something from within + std::string tmp_dir = "__temp__"; + for (unsigned int i = 0; ; ++i) { + if (! tl::file_exists (tl::combine_path (m_local_path, tmp_dir + tl::to_string (i)))) { + tmp_dir += tl::to_string (i); + break; + } + } + auto pc = tl::split (subdir, "/"); + if (! tl::rename_file (tl::combine_path (m_local_path, pc.front ()), tmp_dir)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to rename temp folder"))); + } + pc.front () = tmp_dir; + + if (! tl::mv_dir_recursive (tl::combine_path (m_local_path, tl::join (pc, "/")), m_local_path)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to move subdir components"))); + } + } } diff --git a/src/tl/unit_tests/tlFileUtilsTests.cc b/src/tl/unit_tests/tlFileUtilsTests.cc index f328b28b1..4814901c2 100644 --- a/src/tl/unit_tests/tlFileUtilsTests.cc +++ b/src/tl/unit_tests/tlFileUtilsTests.cc @@ -285,6 +285,58 @@ TEST (3_NOQT) } } +TEST (4_mv_dir) +{ + std::string tmp_dir = tl::absolute_file_path (tmp_file ()); + std::string adir = tl::combine_path (tmp_dir, "a"); + + std::string b1dir = tl::combine_path (adir, "b1"); + tl::mkpath (b1dir); + EXPECT_EQ (tl::file_exists (b1dir), true); + + std::string b2dir = tl::combine_path (adir, "b2"); + tl::mkpath (b2dir); + EXPECT_EQ (tl::file_exists (b1dir), true); + + { + tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "x"))); + os << "hello, world!\n"; + } + + { + tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "y"))); + os << "hello, world II!\n"; + } + + std::string acopydir = tl::combine_path (tmp_dir, "acopy"); + tl::rm_dir_recursive (acopydir); + tl::mkpath (acopydir); + + tl::mv_dir_recursive (adir, acopydir); + + EXPECT_EQ (tl::file_exists (acopydir), true); + EXPECT_EQ (tl::file_exists (adir), false); + + std::string b1copydir = tl::combine_path (acopydir, "b1"); + EXPECT_EQ (tl::file_exists (b1copydir), true); + std::string b2copydir = tl::combine_path (acopydir, "b2"); + EXPECT_EQ (tl::file_exists (b2copydir), true); + + { + std::string xfile = tl::combine_path (b2copydir, "x"); + EXPECT_EQ (tl::file_exists (xfile), true); + tl::InputStream is (xfile); + EXPECT_EQ (is.read_all (), "hello, world!\n"); + } + + { + std::string yfile = tl::combine_path (b2copydir, "y"); + EXPECT_EQ (tl::file_exists (yfile), true); + tl::InputStream is (yfile); + EXPECT_EQ (is.read_all (), "hello, world II!\n"); + } +} + // Secret mode switchers for testing namespace tl { @@ -797,6 +849,13 @@ TEST (18) tl::InputStream is (zfile); EXPECT_EQ (is.read_all (), "hello, world!\n"); } + + // rename directory + tl::rename_file (tl::combine_path (tp, "dir"), "dirx"); + + EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dir")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dirx")), true); + EXPECT_EQ (tl::is_dir (tl::combine_path (tp, "dirx")), true); } // get_home_path diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc index e7e3c1544..86839d7c1 100644 --- a/src/tl/unit_tests/tlGitTests.cc +++ b/src/tl/unit_tests/tlGitTests.cc @@ -24,35 +24,113 @@ #include "tlGit.h" #include "tlUnitTest.h" +#include "tlFileUtils.h" -// @@@ -static std::string test_url ("https://github.com/klayoutmatthias/xsection.git"); +static std::string test_url ("https://github.com/klayout/klayout_git_test.git"); -TEST(1) +TEST(1_plain) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); repo.read (test_url, std::string (), std::string ()); - printf("@@@ done: %s\n", path.c_str ()); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); } -TEST(2) +TEST(2_pathspecs) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("src/**"), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); +} + +TEST(3_subdir) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url + "/src", std::string (), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros/xsection.lym")), true); +} + +TEST(4_single_file) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); repo.read (test_url, std::string ("LICENSE"), std::string ()); - printf("@@@ done: %s\n", path.c_str ()); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src")), false); } -TEST(3) +TEST(5_single_file_from_subdir) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url, std::string (), std::string ("brxxx")); + repo.read (test_url + "/src", std::string ("grain.xml"), std::string ()); - printf("@@@ done: %s\n", path.c_str ()); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.7") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(6_branch) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("wip")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.4") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(7_invalid_branch) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + try { + repo.read (test_url, std::string (), std::string ("brxxx")); + EXPECT_EQ (true, false); + } catch (tl::Exception &ex) { + EXPECT_EQ (ex.msg (), "Error cloning Git repo: reference 'refs/remotes/origin/brxxx' not found"); + } } #endif From 73460016c05a11e3421733dacde82c9dde99cd88 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 25 Oct 2023 23:03:18 +0200 Subject: [PATCH 03/26] Compatibility with old libgit2 --- build.sh | 1 + src/tl/tl/tlGit.cc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/build.sh b/build.sh index a2b7c9280..e6d0d096f 100755 --- a/build.sh +++ b/build.sh @@ -269,6 +269,7 @@ while [ "$*" != "" ]; do echo " -libcurl Use libcurl instead of QtNetwork (for Qt<4.7)" echo " -libexpat Use libexpat instead of QtXml" echo " -libpng Use libpng instead of Qt for PNG generation" + echo " -nolibgit2 Do not include libgit2 for Git package support" echo "" echo "Environment Variables:" echo "" diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index bb0182640..a93282621 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -182,7 +182,11 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons git_repository *cloned_repo = NULL; int error = git_clone (&cloned_repo, url.c_str (), m_local_path.c_str (), &clone_opts); if (error != 0) { +#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28) const git_error *err = git_error_last (); +#else + const git_error *err = giterr_last (); +#endif throw tl::Exception (tl::to_string (tr ("Error cloning Git repo: %s")), (const char *) err->message); } From 2a41b13efd2f0e27fcd864bf94483e1f163acc32 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 25 Oct 2023 23:08:20 +0200 Subject: [PATCH 04/26] Compatibility with old libgit2 --- src/tl/tl/tlGit.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index a93282621..b119f5d59 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -99,7 +99,11 @@ GitObject::~GitObject () } static int +#if LIBGIT2_VER_MAJOR >= 1 fetch_progress (const git_indexer_progress *stats, void *payload) +#else +fetch_progress (const git_transfer_progress *stats, void *payload) +#endif { tl::RelativeProgress *progress = reinterpret_cast (payload); From 4b6cac952724a1e071daff317f6069160d90f12b Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Thu, 26 Oct 2023 00:54:22 +0200 Subject: [PATCH 05/26] Including libgit dependencies in Linux packages --- scripts/makedeb.sh | 8 ++++---- scripts/rpm-data/klayout.spec | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/makedeb.sh b/scripts/makedeb.sh index 81fc8291d..204266666 100755 --- a/scripts/makedeb.sh +++ b/scripts/makedeb.sh @@ -19,16 +19,16 @@ fi # TODO: derive this list automatically? case $target in ubuntu16) - depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)" + depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libgit2-24 (>= 0.24.0), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)" ;; ubuntu18) - depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)" + depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libgit2-26 (>= 0.26.0), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)" ;; ubuntu20) - depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)" + depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libgit2-28 (>= 0.28.4), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)" ;; ubuntu22) - depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)" + depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libgit2-1.1 (>= 1.1.0), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)" ;; *) echo "Unknown target '$target' (given as first argument)" diff --git a/scripts/rpm-data/klayout.spec b/scripts/rpm-data/klayout.spec index 1ad64705f..5a9bad6f2 100644 --- a/scripts/rpm-data/klayout.spec +++ b/scripts/rpm-data/klayout.spec @@ -53,6 +53,7 @@ Requires: qt5-qttools-devel >= 5.11.1 Requires: ruby >= 2.0.0 Requires: python3 >= 3.6.0 Requires: qt-x11 >= 4.8.5 +Requires: libgit2 >= 0.26.8 %define buildopt -j2 %endif @@ -62,7 +63,7 @@ Requires: libcurl >= 7.19.7 Requires: ruby >= 1.8.7 Requires: python >= 2.6.6 Requires: qt-x11 >= 4.6.2 -%define buildopt -libcurl -j2 +%define buildopt -libcurl -j2 -nolibgit2 %endif %if "%{target_system}" == "opensuse42_2" @@ -70,7 +71,7 @@ Requires: qt-x11 >= 4.6.2 Requires: ruby2.3 >= 2.3.1 Requires: python3 >= 3.4.6 Requires: libqt4-x11 >= 4.8.6 -%define buildopt -j2 +%define buildopt -j2 -nolibgit2 %endif %if "%{target_system}" == "opensuse42_3" @@ -78,13 +79,14 @@ Requires: libqt4-x11 >= 4.8.6 Requires: ruby2.3 >= 2.3.1 Requires: python3 >= 3.4.6 Requires: libqt4-x11 >= 4.8.6 -%define buildopt -j2 +%define buildopt -j2 -nolibgit2 %endif %if "%{target_system}" == "opensuse15" # OpenSuSE Leap 15 requirements Requires: ruby >= 2.5 Requires: python3 >= 3.6 +Requires: libgit2 >= 1.3.0 Requires: libqt5-qtbase >= 5.15.2 Requires: libQt5PrintSupport5 >= 5.15.2 Requires: libQt5Designer5 >= 5.15.2 From 2e16a1e3e48172e2caf3d853a7baad366038d384 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 28 Oct 2023 21:37:58 +0200 Subject: [PATCH 06/26] Refined solution for git clone: now accepts tags too. --- src/tl/tl/tlGit.cc | 179 ++++++++++++++++++++++++++------ src/tl/unit_tests/tlGitTests.cc | 70 ++++++++++++- 2 files changed, 216 insertions(+), 33 deletions(-) diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index b119f5d59..9479ba5a8 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -25,6 +25,7 @@ #include "tlFileUtils.h" #include "tlProgress.h" #include "tlStaticObjects.h" +#include "tlLog.h" #include #include @@ -124,6 +125,101 @@ checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload) progress->set (count + 5000u); } +static void check (int error) +{ + if (error != 0) { +#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28) + const git_error *err = git_error_last (); +#else + const git_error *err = giterr_last (); +#endif + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo: %s")), (const char *) err->message); + } +} + +static bool +ref_matches (const char *name, const std::string &ref) +{ + if (!name) { + return false; + } else if (name == ref) { + return true; + } else if (name == "refs/heads/" + ref) { + return true; + } else if (name == "refs/tags/" + ref) { + return true; + } else { + return false; + } +} + +static void +checkout_branch (git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch) +{ + git_buf remote_branch = GIT_BUF_INIT_CONST (NULL, 0); + + try { + + git_oid oid; + + // if no branch is given, use the default branch + if (! branch) { + check (git_remote_default_branch (&remote_branch, remote)); + branch = remote_branch.ptr; + if (tl::verbosity () >= 10) { + tl::info << tr ("Git checkout: Using default branch for repository ") << git_remote_url (remote) << ": " << branch; + } + } else { + if (tl::verbosity () >= 10) { + tl::info << tr ("Git checkout: Checking out branch for repository ") << git_remote_url (remote) << ": " << branch; + } + } + + // resolve the branch by using ls-remote: + + size_t n = 0; + const git_remote_head **ls = NULL; + check (git_remote_ls (&ls, &n, remote)); + + if (tl::verbosity () >= 20) { + tl::info << "Git checkout: ls-remote on " << git_remote_url (remote) << ":"; + } + + bool found = false; + + for (size_t i = 0; i < n; ++i) { + const git_remote_head *rh = ls[i]; + if (tl::verbosity () >= 20) { + char oid_fmt [80]; + git_oid_tostr (oid_fmt, sizeof (oid_fmt), &rh->oid); + tl::info << " " << rh->name << ": " << (const char *) oid_fmt; + } + if (ref_matches (rh->name, branch)) { + oid = rh->oid; + found = true; + } + } + + if (! found) { + throw tl::Exception (tl::to_string (tr ("Git checkout - Unable to resolve reference name: ")) + branch); + } + + if (tl::verbosity () >= 10) { + char oid_fmt [80]; + git_oid_tostr (oid_fmt, sizeof (oid_fmt), &oid); + tl::info << tr ("Git checkout: resolving ") << branch << tr (" to ") << (const char *) oid_fmt; + } + + check (git_repository_set_head_detached (repo, &oid)); + check (git_checkout_head (repo, co_opts)); + + } catch (...) { + git_buf_dispose (&remote_branch); + throw; + } + + git_buf_dispose (&remote_branch); +} void GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) @@ -151,6 +247,8 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons // @@@ use callback, timeout? tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/); + // build checkout options + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; const char *paths_cstr[1]; @@ -160,49 +258,68 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons checkout_opts.paths.strings = (char **) &paths_cstr; } - /* - checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE | - GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; - @@@*/ checkout_opts.progress_cb = &checkout_progress; checkout_opts.progress_payload = (void *) &progress; - git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + // build fetch options - clone_opts.checkout_opts = checkout_opts; + git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; - // NOTE: really has to be a branch! Tags won't work. - if (! branch.empty ()) { - clone_opts.checkout_branch = branch.c_str (); - } + fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_AUTO; #if LIBGIT2_VER_MAJOR > 1 || (LIBGIT2_VER_MAJOR == 1 && LIBGIT2_VER_MINOR >= 7) - clone_opts.fetch_opts.depth = 1; // shallow (single commit) + fetch_opts.depth = 1; // shallow (single commit) #endif - clone_opts.fetch_opts.callbacks.transfer_progress = &fetch_progress; - clone_opts.fetch_opts.callbacks.payload = (void *) &progress; + fetch_opts.callbacks.transfer_progress = &fetch_progress; + fetch_opts.callbacks.payload = (void *) &progress; + + // build refspecs in case they are needed + + char *refs[] = { (char *) branch.c_str () }; + git_strarray refspecs; + refspecs.count = 1; + refspecs.strings = refs; + git_strarray *refspecs_p = branch.empty () ? NULL : &refspecs; + + // Make repository - // Do the clone git_repository *cloned_repo = NULL; - int error = git_clone (&cloned_repo, url.c_str (), m_local_path.c_str (), &clone_opts); - if (error != 0) { -#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28) - const git_error *err = git_error_last (); -#else - const git_error *err = giterr_last (); -#endif - throw tl::Exception (tl::to_string (tr ("Error cloning Git repo: %s")), (const char *) err->message); + git_remote *remote = NULL; + + try { + + check (git_repository_init (&cloned_repo, m_local_path.c_str (), 0)); + + check (git_remote_create (&remote, cloned_repo, "download", url.c_str ())); + + // actually fetch + if (tl::verbosity () >= 10) { + tl::info << tr ("Fetching Git repo from ") << git_remote_url (remote) << " ..."; + } + check (git_remote_fetch (remote, refspecs_p, &fetch_opts, NULL)); + + // checkout + checkout_branch (cloned_repo, remote, &checkout_opts, branch.empty () ? 0 : branch.c_str ()); + + // free the repo and remote + git_repository_free (cloned_repo); + git_remote_free (remote); + + // get rid of ".git" - we do not need it anymore + + tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + + } catch (...) { + // free the repo in the error case + if (cloned_repo != NULL) { + git_repository_free (cloned_repo); + } + if (remote != NULL) { + git_remote_free (remote); + } + throw; } - if (! cloned_repo) { - throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - no data available"))); - } - - git_repository_free (cloned_repo); - - // remove the worktree as we don't need it - tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); - // pull subfolder files to target path level if (! subdir.empty ()) { diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc index 86839d7c1..cd9c5fd9e 100644 --- a/src/tl/unit_tests/tlGitTests.cc +++ b/src/tl/unit_tests/tlGitTests.cc @@ -121,7 +121,73 @@ TEST(6_branch) EXPECT_EQ (found, true); } -TEST(7_invalid_branch) +TEST(7_tag) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("1.2")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.2") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(8_refspec) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("refs/tags/1.5")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.5") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(9_HEAD) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("HEAD")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.7") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(10_invalid_branch) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); @@ -129,7 +195,7 @@ TEST(7_invalid_branch) repo.read (test_url, std::string (), std::string ("brxxx")); EXPECT_EQ (true, false); } catch (tl::Exception &ex) { - EXPECT_EQ (ex.msg (), "Error cloning Git repo: reference 'refs/remotes/origin/brxxx' not found"); + EXPECT_EQ (ex.msg (), "Git checkout - Unable to resolve reference name: brxxx"); } } From ba19b3374afd610e793f6b44c27b29594a6c9a83 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 28 Oct 2023 21:54:56 +0200 Subject: [PATCH 07/26] Fixed build issue with old versions of libgit2 --- src/tl/tl/tlGit.cc | 137 ++++++++++++++++++++++++++------------------- src/tl/tl/tlGit.h | 3 + 2 files changed, 82 insertions(+), 58 deletions(-) diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 9479ba5a8..13d71d123 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -153,72 +153,93 @@ ref_matches (const char *name, const std::string &ref) } } +namespace +{ + +class GitBuffer +{ +public: + GitBuffer () + { + m_buf = GIT_BUF_INIT_CONST (NULL, 0); + } + + ~GitBuffer () + { +#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28) + git_buf_dispose (&m_buf); +#else + git_buf_free (&m_buf); +#endif + } + + const char *c_str () const { return m_buf.ptr; } + + git_buf *get () { return &m_buf; } + const git_buf *get () const { return &m_buf; } + +private: + git_buf m_buf; +}; + +} + static void checkout_branch (git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch) { - git_buf remote_branch = GIT_BUF_INIT_CONST (NULL, 0); - - try { - - git_oid oid; - - // if no branch is given, use the default branch - if (! branch) { - check (git_remote_default_branch (&remote_branch, remote)); - branch = remote_branch.ptr; - if (tl::verbosity () >= 10) { - tl::info << tr ("Git checkout: Using default branch for repository ") << git_remote_url (remote) << ": " << branch; - } - } else { - if (tl::verbosity () >= 10) { - tl::info << tr ("Git checkout: Checking out branch for repository ") << git_remote_url (remote) << ": " << branch; - } - } - - // resolve the branch by using ls-remote: - - size_t n = 0; - const git_remote_head **ls = NULL; - check (git_remote_ls (&ls, &n, remote)); - - if (tl::verbosity () >= 20) { - tl::info << "Git checkout: ls-remote on " << git_remote_url (remote) << ":"; - } - - bool found = false; - - for (size_t i = 0; i < n; ++i) { - const git_remote_head *rh = ls[i]; - if (tl::verbosity () >= 20) { - char oid_fmt [80]; - git_oid_tostr (oid_fmt, sizeof (oid_fmt), &rh->oid); - tl::info << " " << rh->name << ": " << (const char *) oid_fmt; - } - if (ref_matches (rh->name, branch)) { - oid = rh->oid; - found = true; - } - } - - if (! found) { - throw tl::Exception (tl::to_string (tr ("Git checkout - Unable to resolve reference name: ")) + branch); - } + GitBuffer remote_branch; + git_oid oid; + // if no branch is given, use the default branch + if (! branch) { + check (git_remote_default_branch (remote_branch.get (), remote)); + branch = remote_branch.c_str (); if (tl::verbosity () >= 10) { - char oid_fmt [80]; - git_oid_tostr (oid_fmt, sizeof (oid_fmt), &oid); - tl::info << tr ("Git checkout: resolving ") << branch << tr (" to ") << (const char *) oid_fmt; + tl::info << tr ("Git checkout: Using default branch for repository ") << git_remote_url (remote) << ": " << branch; + } + } else { + if (tl::verbosity () >= 10) { + tl::info << tr ("Git checkout: Checking out branch for repository ") << git_remote_url (remote) << ": " << branch; } - - check (git_repository_set_head_detached (repo, &oid)); - check (git_checkout_head (repo, co_opts)); - - } catch (...) { - git_buf_dispose (&remote_branch); - throw; } - git_buf_dispose (&remote_branch); + // resolve the branch by using ls-remote: + + size_t n = 0; + const git_remote_head **ls = NULL; + check (git_remote_ls (&ls, &n, remote)); + + if (tl::verbosity () >= 20) { + tl::info << "Git checkout: ls-remote on " << git_remote_url (remote) << ":"; + } + + bool found = false; + + for (size_t i = 0; i < n; ++i) { + const git_remote_head *rh = ls[i]; + if (tl::verbosity () >= 20) { + char oid_fmt [80]; + git_oid_tostr (oid_fmt, sizeof (oid_fmt), &rh->oid); + tl::info << " " << rh->name << ": " << (const char *) oid_fmt; + } + if (ref_matches (rh->name, branch)) { + oid = rh->oid; + found = true; + } + } + + if (! found) { + throw tl::Exception (tl::to_string (tr ("Git checkout - Unable to resolve reference name: ")) + branch); + } + + if (tl::verbosity () >= 10) { + char oid_fmt [80]; + git_oid_tostr (oid_fmt, sizeof (oid_fmt), &oid); + tl::info << tr ("Git checkout: resolving ") << branch << tr (" to ") << (const char *) oid_fmt; + } + + check (git_repository_set_head_detached (repo, &oid)); + check (git_checkout_head (repo, co_opts)); } void diff --git a/src/tl/tl/tlGit.h b/src/tl/tl/tlGit.h index cbc11bda9..68a1db83a 100644 --- a/src/tl/tl/tlGit.h +++ b/src/tl/tl/tlGit.h @@ -81,6 +81,9 @@ public: * * This method throws an exception if the directory structure could * not be obtained or downloading of one file failed. + * + * "branch" is the remote ref to use. This can be a branch name, a tag name, + * a remote ref such as "refs/heads/master" or a symbolic name such as "HEAD". */ static bool download (const std::string &url, const std::string &target, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); From b56220d36dd12a4a992d00a67f374b8fb26bead5 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 28 Oct 2023 23:48:56 +0200 Subject: [PATCH 08/26] [consider merging] Functional ut_runner binary also with -without-qt --- src/db/db/db.pro | 2 +- src/drc/drc/drc.pro | 2 +- src/lib/lib/lib.pro | 82 ++++++++++++++++--------------- src/lvs/lvs/lvs.pro | 2 +- src/rba/unit_tests/unit_tests.pro | 2 +- src/unit_tests/unit_test_main.cc | 4 +- 6 files changed, 48 insertions(+), 46 deletions(-) diff --git a/src/db/db/db.pro b/src/db/db/db.pro index 6914e12fa..513cca3a0 100644 --- a/src/db/db/db.pro +++ b/src/db/db/db.pro @@ -404,7 +404,7 @@ HEADERS = \ dbShapeCollection.h \ dbShapeCollectionUtils.h -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ dbResources.qrc \ diff --git a/src/drc/drc/drc.pro b/src/drc/drc/drc.pro index c43e875d0..2304dc132 100644 --- a/src/drc/drc/drc.pro +++ b/src/drc/drc/drc.pro @@ -13,7 +13,7 @@ HEADERS = \ drcCommon.h \ drcForceLink.h \ -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ drcResources.qrc } diff --git a/src/lib/lib/lib.pro b/src/lib/lib/lib.pro index 8352f19af..98f84bd54 100644 --- a/src/lib/lib/lib.pro +++ b/src/lib/lib/lib.pro @@ -1,40 +1,42 @@ - -DESTDIR = $$OUT_PWD/../.. -TARGET = klayout_lib - -include($$PWD/../../lib.pri) - -DEFINES += MAKE_LIB_LIBRARY - -HEADERS = \ - libBasicArc.h \ - libBasicCircle.h \ - libBasicDonut.h \ - libBasicEllipse.h \ - libBasicPie.h \ - libBasicRoundPath.h \ - libBasicRoundPolygon.h \ - libBasicStrokedPolygon.h \ - libBasicText.h \ - libForceLink.h - -SOURCES = \ - libForceLink.cc \ - libBasic.cc \ - libBasicArc.cc \ - libBasicCircle.cc \ - libBasicDonut.cc \ - libBasicEllipse.cc \ - libBasicPie.cc \ - libBasicRoundPath.cc \ - libBasicRoundPolygon.cc \ - libBasicStrokedPolygon.cc \ - libBasicText.cc - -RESOURCES = \ - libResources.qrc - -INCLUDEPATH += $$TL_INC $$GSI_INC $$DB_INC -DEPENDPATH += $$TL_INC $$GSI_INC $$DB_INC -LIBS += -L$$DESTDIR -lklayout_gsi -lklayout_tl -lklayout_db - + +DESTDIR = $$OUT_PWD/../.. +TARGET = klayout_lib + +include($$PWD/../../lib.pri) + +DEFINES += MAKE_LIB_LIBRARY + +HEADERS = \ + libBasicArc.h \ + libBasicCircle.h \ + libBasicDonut.h \ + libBasicEllipse.h \ + libBasicPie.h \ + libBasicRoundPath.h \ + libBasicRoundPolygon.h \ + libBasicStrokedPolygon.h \ + libBasicText.h \ + libForceLink.h + +SOURCES = \ + libForceLink.cc \ + libBasic.cc \ + libBasicArc.cc \ + libBasicCircle.cc \ + libBasicDonut.cc \ + libBasicEllipse.cc \ + libBasicPie.cc \ + libBasicRoundPath.cc \ + libBasicRoundPolygon.cc \ + libBasicStrokedPolygon.cc \ + libBasicText.cc + +!equals(HAVE_QT, "0") { + RESOURCES = \ + libResources.qrc +} + +INCLUDEPATH += $$TL_INC $$GSI_INC $$DB_INC +DEPENDPATH += $$TL_INC $$GSI_INC $$DB_INC +LIBS += -L$$DESTDIR -lklayout_gsi -lklayout_tl -lklayout_db + diff --git a/src/lvs/lvs/lvs.pro b/src/lvs/lvs/lvs.pro index 5a8c53f27..722270f84 100644 --- a/src/lvs/lvs/lvs.pro +++ b/src/lvs/lvs/lvs.pro @@ -13,7 +13,7 @@ HEADERS = \ lvsCommon.h \ lvsForceLink.h \ -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ lvsResources.qrc } diff --git a/src/rba/unit_tests/unit_tests.pro b/src/rba/unit_tests/unit_tests.pro index 6bca18f2f..5924ccae6 100644 --- a/src/rba/unit_tests/unit_tests.pro +++ b/src/rba/unit_tests/unit_tests.pro @@ -14,7 +14,7 @@ DEPENDPATH += $$RBA_INC $$TL_INC $$DB_INC $$GSI_INC LIBS += -L$$DESTDIR_UT -lklayout_rba -lklayout_tl -lklayout_db -lklayout_gsi -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ rba_unit_tests.qrc } diff --git a/src/unit_tests/unit_test_main.cc b/src/unit_tests/unit_test_main.cc index 75c8f7809..2837663bd 100644 --- a/src/unit_tests/unit_test_main.cc +++ b/src/unit_tests/unit_test_main.cc @@ -433,6 +433,8 @@ run_tests (const std::vector &selected_tests, bool editable, boo static int main_cont (int &argc, char **argv) { + ut::TestConsole console (stdout); + std::unique_ptr ruby_interpreter; std::unique_ptr python_interpreter; @@ -452,8 +454,6 @@ main_cont (int &argc, char **argv) int result = 0; - ut::TestConsole console (stdout); - try { pya::PythonInterpreter::initialize (); From 2ed44e27ad6f11fe7c3242b7e4ea6100e5e11d3a Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 28 Oct 2023 23:49:50 +0200 Subject: [PATCH 09/26] Git support: default checkout strategy to 'force' to support older versions of libgit2 --- src/tl/tl/tlGit.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 13d71d123..3279bef16 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -271,6 +271,7 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons // build checkout options git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE; const char *paths_cstr[1]; paths_cstr[0] = filter.c_str (); From bd785279ef69f24a56b63d020c282e63e775725a Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 29 Oct 2023 15:30:08 +0100 Subject: [PATCH 10/26] A helper class for parsing Salt Grain URLs into protocol, branch, subfolder --- src/lay/lay/lay.pro | 2 + src/lay/lay/laySaltParsedURL.cc | 149 ++++++++++++++++++++ src/lay/lay/laySaltParsedURL.h | 106 ++++++++++++++ src/lay/unit_tests/laySaltParsedURLTests.cc | 132 +++++++++++++++++ src/lay/unit_tests/unit_tests.pro | 1 + 5 files changed, 390 insertions(+) create mode 100644 src/lay/lay/laySaltParsedURL.cc create mode 100644 src/lay/lay/laySaltParsedURL.h create mode 100644 src/lay/unit_tests/laySaltParsedURLTests.cc diff --git a/src/lay/lay/lay.pro b/src/lay/lay/lay.pro index 3500502b8..23295c1f8 100644 --- a/src/lay/lay/lay.pro +++ b/src/lay/lay/lay.pro @@ -33,6 +33,7 @@ HEADERS = \ layResourceHelpProvider.h \ layRuntimeErrorForm.h \ layReaderErrorForm.h \ + laySaltParsedURL.h \ laySearchReplaceConfigPage.h \ laySearchReplaceDialog.h \ laySearchReplacePropertiesWidgets.h \ @@ -144,6 +145,7 @@ SOURCES = \ layResourceHelpProvider.cc \ layRuntimeErrorForm.cc \ layReaderErrorForm.cc \ + laySaltParsedURL.cc \ laySearchReplaceConfigPage.cc \ laySearchReplaceDialog.cc \ laySearchReplacePlugin.cc \ diff --git a/src/lay/lay/laySaltParsedURL.cc b/src/lay/lay/laySaltParsedURL.cc new file mode 100644 index 000000000..8df57de5f --- /dev/null +++ b/src/lay/lay/laySaltParsedURL.cc @@ -0,0 +1,149 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "laySaltParsedURL.h" +#include "laySaltGrain.h" +#include "tlString.h" + +namespace lay +{ + +static void +parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::string &subfolder) +{ + const char *org_str = ex.get (); + + std::string w; + // protocol (http:) + ex.try_read_word (w, "") && ex.test (":"); + while (! ex.at_end () && ex.test ("/")) { + ; + } + // server ("www.klayout.de") + while (! ex.at_end () && (*ex != '/' && *ex != '+')) { + ++ex; + } + + while (! ex.at_end ()) { + + ++ex; + + const char *c1 = ex.get (); + while (! ex.at_end () && (*ex != '/' && *ex != '+' && *ex != '[')) { + ++ex; + } + const char *c2 = ex.get (); + + std::string comp (c1, c2 - c1); + + if ((! ex.at_end () && *ex == '+') || comp.find (".git") == comp.size () - 4) { + // subfolder starts here + break; + } + + } + + url = std::string (org_str, ex.get () - org_str); + + if (ex.at_end ()) { + return; + } + + if (*ex == '/') { + while (! ex.at_end () && *ex == '/') { + ++ex; + } + } else if (*ex == '+') { + ++ex; + } + + { + const char *c1 = ex.get (); + while (! ex.at_end () && *ex != '[') { + ++ex; + } + const char *c2 = ex.get (); + subfolder = std::string (c1, c2 - c1); + } + + if (! ex.at_end () && *ex == '[') { + + // explicit branch + ++ex; + const char *c1 = ex.get (); + while (! ex.at_end () && *ex != ']') { + ++ex; + } + const char *c2 = ex.get (); + branch = std::string (c1, c2 - c1); + + } else if (! subfolder.empty ()) { + + // SVN emulation + + auto parts = tl::split (subfolder, "/"); + if (parts.size () >= 1 && parts.back () == "trunk") { + + branch = "HEAD"; + parts.pop_back (); + subfolder = tl::join (parts, "/"); + + } else if (parts.size () >= 2 && parts[parts.size () - 2] == "tags") { + + branch = "refs/tags/" + parts.back (); + parts.pop_back (); + parts.pop_back (); + subfolder = tl::join (parts, "/"); + + } else if (parts.size () >= 2 && parts[parts.size () - 2] == "branches") { + + branch = "refs/heads/" + parts.back (); + parts.pop_back (); + parts.pop_back (); + subfolder = tl::join (parts, "/"); + + } + + } +} + +SaltParsedURL::SaltParsedURL (const std::string &url) + : m_protocol (lay::DefaultProtocol) +{ + tl::Extractor ex (url.c_str ()); + if (ex.test ("svn") && ex.test ("+")) { + m_protocol = lay::WebDAV; + m_url = ex.get (); + return; + } + + ex = tl::Extractor (url.c_str ()); + if (ex.test ("git") && ex.test ("+")) { + m_protocol = lay::Git; + parse_git_url (ex, m_url, m_branch, m_subfolder); + return; + } + + m_url = url; +} + +} diff --git a/src/lay/lay/laySaltParsedURL.h b/src/lay/lay/laySaltParsedURL.h new file mode 100644 index 000000000..bc076f76b --- /dev/null +++ b/src/lay/lay/laySaltParsedURL.h @@ -0,0 +1,106 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef HDR_laySaltParsedURL +#define HDR_laySaltParsedURL + +#include "layCommon.h" +#include "laySaltGrain.h" + +namespace lay +{ + +/** + * @brief A class representing a SaltGrain URL + * + * The URL is parsed into protocol, branch, URL and subfolder if applicable. + * Some heuristics is applied to decompose parts. + * + * SVN URLs: + * https://server.com/repo/trunk -> protocol=DefaultProtocol, url="https://server.com/repo/trunk", branch="", subfolder="" + * svn+https://server.com/repo/trunk -> protocol=WebDAV, url="https://server.com/repo/trunk", branch="", subfolder="" + * + * Git URL heuristics: + * git+https://server.com/repo.git -> protocol=Git, url="https://server.com/repo.git", branch="", subfolder="" + * git+https://server.com/repo.git/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="", subfolder="sub/folder" + * git+https://server.com/repo+sub/folder -> protocol=Git, url="https://server.com/repo", branch="", subfolder="sub/folder" + * git+https://server.com/repo.git[v1.0] -> protocol=Git, url="https://server.com/repo.git", branch="v1.0", subfolder="" + * git+https://server.com/repo.git/sub/folder[refs/tags/1.0] -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.0", subfolder="sub/folder" + * git+https://server.com/repo.git/trunk -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="" + * git+https://server.com/repo.git/sub/folder/trunk -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="sub/folder" + * git+https://server.com/repo.git/branches/release -> protocol=Git, url="https://server.com/repo.git", branch="refs/heads/release", subfolder="" + * git+https://server.com/repo.git/tags/1.9 -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="" + * git+https://server.com/repo.git/sub/folder/tags/1.9 -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="sub/folder" + */ + +class LAY_PUBLIC SaltParsedURL +{ +public: + /** + * @brief Constructor: creates an URL from the given generic URL string + * + * This will decompose the URL into the parts and fill protocol, branch and subfolder fields. + */ + SaltParsedURL (const std::string &url); + + /** + * @brief Gets the basic URL + */ + const std::string &url () const + { + return m_url; + } + + /** + * @brief Gets the subfolder string + */ + const std::string &subfolder () const + { + return m_subfolder; + } + + /** + * @brief Gets the branch string + */ + const std::string &branch () const + { + return m_branch; + } + + /** + * @brief Gets the protocol + */ + lay::Protocol protocol () const + { + return m_protocol; + } + +private: + std::string m_url; + std::string m_branch; + std::string m_subfolder; + lay::Protocol m_protocol; +}; + +} + +#endif diff --git a/src/lay/unit_tests/laySaltParsedURLTests.cc b/src/lay/unit_tests/laySaltParsedURLTests.cc new file mode 100644 index 000000000..3b4985d88 --- /dev/null +++ b/src/lay/unit_tests/laySaltParsedURLTests.cc @@ -0,0 +1,132 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "laySaltParsedURL.h" +#include "tlUnitTest.h" + +TEST (1_Basic) +{ + lay::SaltParsedURL purl ("https://server.com/repo/trunk"); + EXPECT_EQ (purl.protocol () == lay::DefaultProtocol, true); + EXPECT_EQ (purl.url (), "https://server.com/repo/trunk"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (2_SVN) +{ + lay::SaltParsedURL purl ("svn+https://server.com/repo/trunk"); + EXPECT_EQ (purl.protocol () == lay::WebDAV, true); + EXPECT_EQ (purl.url (), "https://server.com/repo/trunk"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (10_GitBasic) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (11_GitSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (12_GitExplicitBranch) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git[v1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "v1.0"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (13_GitExplicitBranchAndSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder[refs/tags/1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/tags/1.0"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (14_GitExplicitBranchAndExplicitSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo+sub/folder[refs/tags/1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo"); + EXPECT_EQ (purl.branch (), "refs/tags/1.0"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (15_GitSVNEmulationTrunk) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/trunk"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "HEAD"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (16_GitSVNEmulationTrunkWithSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder/trunk"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "HEAD"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (17_GitSVNEmulationBranch) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/branches/xyz"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/heads/xyz"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (18_GitSVNEmulationTag) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/tags/1.9"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/tags/1.9"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (19_GitSVNEmulationTagWithSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder/tags/1.9"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/tags/1.9"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} diff --git a/src/lay/unit_tests/unit_tests.pro b/src/lay/unit_tests/unit_tests.pro index bd437bdbb..adf955966 100644 --- a/src/lay/unit_tests/unit_tests.pro +++ b/src/lay/unit_tests/unit_tests.pro @@ -9,6 +9,7 @@ include($$PWD/../../lib_ut.pri) SOURCES = \ laySalt.cc \ layHelpIndexTest.cc \ + laySaltParsedURLTests.cc \ laySessionTests.cc INCLUDEPATH += $$LAY_INC $$TL_INC $$LAYBASIC_INC $$LAYUI_INC $$LAYVIEW_INC $$DB_INC $$GSI_INC $$ANT_INC $$IMG_INC $$RDB_INC From 6dec3b0348917c85a7399a7b8e96316c63077064 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 29 Oct 2023 16:04:12 +0100 Subject: [PATCH 11/26] Using parsed URLs for packages --- src/lay/lay/laySalt.cc | 9 ++- src/lay/lay/laySaltController.cc | 35 ++-------- src/lay/lay/laySaltDownloadManager.cc | 30 +++------ src/lay/lay/laySaltDownloadManager.h | 10 ++- src/lay/lay/laySaltGrain.cc | 67 +++++--------------- src/lay/lay/laySaltGrain.h | 46 +------------- src/lay/lay/laySaltGrainDetailsTextWidget.cc | 1 + src/lay/lay/laySaltGrainPropertiesDialog.cc | 5 +- src/lay/lay/laySaltManagerDialog.cc | 8 +-- src/lay/lay/laySaltParsedURL.h | 9 +++ src/tl/tl/tlGit.cc | 33 ++++------ src/tl/tl/tlGit.h | 6 +- src/tl/unit_tests/tlGitTests.cc | 44 +++++++------ 13 files changed, 98 insertions(+), 205 deletions(-) diff --git a/src/lay/lay/laySalt.cc b/src/lay/lay/laySalt.cc index 4f5f428b7..2abf3c519 100644 --- a/src/lay/lay/laySalt.cc +++ b/src/lay/lay/laySalt.cc @@ -21,6 +21,7 @@ */ #include "laySalt.h" +#include "laySaltParsedURL.h" #include "tlString.h" #include "tlFileUtils.h" @@ -492,16 +493,18 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, double timeout, t // otherwise download from the URL using Git or SVN - if (templ.protocol () == Git) { + lay::SaltParsedURL purl (templ.url ()); + + if (purl.protocol () == Git) { #if defined(HAVE_GIT2) tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); - res = tl::GitObject::download (templ.url (), target.path (), templ.branch (), timeout, callback); + res = tl::GitObject::download (templ.url (), target.path (), purl.subfolder (), purl.branch (), timeout, callback); #else throw tl::Exception (tl::to_string (QObject::tr ("Unable to install package '%1' - git protocol not compiled in").arg (tl::to_qstring (target.name ())))); #endif - } else if (templ.protocol () == WebDAV || templ.protocol () == DefaultProtocol) { + } else if (purl.protocol () == WebDAV || purl.protocol () == DefaultProtocol) { tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback); diff --git a/src/lay/lay/laySaltController.cc b/src/lay/lay/laySaltController.cc index 2a10a8894..efd85a58e 100644 --- a/src/lay/lay/laySaltController.cc +++ b/src/lay/lay/laySaltController.cc @@ -23,6 +23,7 @@ #include "laySaltController.h" #include "laySaltManagerDialog.h" #include "laySaltDownloadManager.h" +#include "laySaltParsedURL.h" #include "layConfig.h" #include "layMainWindow.h" #include "layQtTools.h" @@ -202,40 +203,18 @@ SaltController::install_packages (const std::vector &packages, bool } } - if (n.find ("http:") == 0 || n.find ("https:") == 0 || n.find ("file:") == 0 || n[0] == '/' || n[0] == '\\') { + lay::SaltParsedURL purl (n); + const std::string &url = purl.url (); + + if (url.find ("http:") == 0 || url.find ("https:") == 0 || url.find ("file:") == 0 || url[0] == '/' || url[0] == '\\') { // its a URL - manager.register_download (std::string (), std::string (), n, DefaultProtocol, std::string (), v); - - } else if (n.find ("git@") == 0) { - - // git protocol: - // "git@" - // "git@[]" - - std::string url (n, 4); - size_t br = url.find ("["); - std::string branch; - if (br != std::string::npos && url.back () == ']') { - branch = std::string (url, br + 1, url.size () - br - 2); - url = std::string (url, 0, br); - } - - manager.register_download (std::string (), std::string (), url, Git, branch, v); - - } else if (n.find ("svn@") == 0) { - - // svn protocol: - // "svn@" - - std::string url (n, 4); - // its a URL - manager.register_download (std::string (), std::string (), url, WebDAV, std::string (), v); + manager.register_download (std::string (), std::string (), n, v); } else { // its a plain name - manager.register_download (n, std::string (), std::string (), DefaultProtocol, std::string (), v); + manager.register_download (n, std::string (), std::string (), v); } diff --git a/src/lay/lay/laySaltDownloadManager.cc b/src/lay/lay/laySaltDownloadManager.cc index 4d9032f6c..cc0b625d7 100644 --- a/src/lay/lay/laySaltDownloadManager.cc +++ b/src/lay/lay/laySaltDownloadManager.cc @@ -58,7 +58,7 @@ ConfirmationDialog::ConfirmationDialog (QWidget *parent) } void -ConfirmationDialog::add_info (const std::string &name, bool update, const std::string &version, const std::string &url, Protocol protocol, const std::string &branch) +ConfirmationDialog::add_info (const std::string &name, bool update, const std::string &version, const std::string &url) { QTreeWidgetItem *item = new QTreeWidgetItem (list); m_items_by_name.insert (std::make_pair (name, item)); @@ -68,18 +68,7 @@ ConfirmationDialog::add_info (const std::string &name, bool update, const std::s item->setText (0, tl::to_qstring (name)); item->setText (1, update ? tr ("UPDATE") : tr ("INSTALL")); item->setText (2, tl::to_qstring (version)); - - if (protocol == WebDAV) { - item->setText (3, tl::to_qstring ("svn@" + url)); - } else if (protocol == Git) { - if (branch.empty ()) { - item->setText (3, tl::to_qstring ("git@" + url)); - } else { - item->setText (3, tl::to_qstring ("git@" + url + "[" + branch + "]")); - } - } else { - item->setText (3, tl::to_qstring (url)); - } + item->setText (3, tl::to_qstring (url)); for (int column = 0; column < list->colorCount (); ++column) { item->setData (column, Qt::ForegroundRole, QVariant (QBrush (update ? QColor (Qt::blue) : QColor (Qt::black)))); @@ -180,9 +169,9 @@ SaltDownloadManager::SaltDownloadManager () } void -SaltDownloadManager::register_download (const std::string &name, const std::string &token, const std::string &url, Protocol protocol, const std::string &branch, const std::string &version) +SaltDownloadManager::register_download (const std::string &name, const std::string &token, const std::string &url, const std::string &version) { - m_registry.push_back (Descriptor (name, token, url, protocol, branch, version)); + m_registry.push_back (Descriptor (name, token, url, version)); } void @@ -255,7 +244,7 @@ SaltDownloadManager::compute_list (const lay::Salt &salt, const lay::Salt &salt_ if (tl::verbosity() >= 20) { tl::log << "Considering for update as dependency: " << d->name << " (" << d->version << ") with URL " << d->url; } - m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->protocol, d->branch, d->version)); + m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->version)); } else { if (tl::verbosity() >= 20) { @@ -268,7 +257,7 @@ SaltDownloadManager::compute_list (const lay::Salt &salt, const lay::Salt &salt_ if (tl::verbosity() >= 20) { tl::log << "Considering for download as dependency: " << d->name << " (" << d->version << ") with URL " << d->url; } - m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->protocol, d->branch, d->version)); + m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->version)); } @@ -341,7 +330,8 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt } try { - p->grain = SaltGrain::from_url (p->url, p->protocol, p->branch); + // @@@ Take from repo index for Git protocol? + p->grain = SaltGrain::from_url (p->url); } catch (tl::Exception &ex) { throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ())))); } @@ -398,7 +388,7 @@ SaltDownloadManager::make_confirmation_dialog (QWidget *parent, const lay::Salt const lay::SaltGrain *g = salt.grain_by_name (p->name); if (g) { // \342\206\222 is UTF-8 "right arrow" - dialog->add_info (p->name, true, g->version () + " \342\206\222 " + p->version, p->url, p->protocol, p->branch); + dialog->add_info (p->name, true, g->version () + " \342\206\222 " + p->version, p->url); } } @@ -406,7 +396,7 @@ SaltDownloadManager::make_confirmation_dialog (QWidget *parent, const lay::Salt for (std::vector::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) { const lay::SaltGrain *g = salt.grain_by_name (p->name); if (!g) { - dialog->add_info (p->name, false, p->version, p->url, p->protocol, p->branch); + dialog->add_info (p->name, false, p->version, p->url); } } diff --git a/src/lay/lay/laySaltDownloadManager.h b/src/lay/lay/laySaltDownloadManager.h index 04c3ef2b5..a81e0f973 100644 --- a/src/lay/lay/laySaltDownloadManager.h +++ b/src/lay/lay/laySaltDownloadManager.h @@ -53,7 +53,7 @@ Q_OBJECT public: ConfirmationDialog (QWidget *parent); - void add_info (const std::string &name, bool update, const std::string &version, const std::string &url, Protocol protocol, const std::string &branch); + void add_info (const std::string &name, bool update, const std::string &version, const std::string &url); bool is_confirmed () const { return m_confirmed; } bool is_cancelled () const { return m_cancelled; } @@ -108,7 +108,7 @@ public: * * The target directory can be empty. In this case, the downloader will pick an appropriate one. */ - void register_download (const std::string &name, const std::string &token, const std::string &url, Protocol protocol, const std::string &branch, const std::string &version); + void register_download (const std::string &name, const std::string &token, const std::string &url, const std::string &version); /** * @brief Computes the dependencies after all required packages have been registered @@ -145,8 +145,8 @@ public: private: struct Descriptor { - Descriptor (const std::string &_name, const std::string &_token, const std::string &_url, Protocol _protocol, const std::string &_branch, const std::string &_version) - : name (_name), token (_token), url (_url), protocol (_protocol), branch (_branch), version (_version), downloaded (false) + Descriptor (const std::string &_name, const std::string &_token, const std::string &_url, const std::string &_version) + : name (_name), token (_token), url (_url), version (_version), downloaded (false) { } bool operator< (const Descriptor &other) const @@ -170,8 +170,6 @@ private: std::string name; std::string token; std::string url; - Protocol protocol; - std::string branch; std::string version; bool downloaded; lay::SaltGrain grain; diff --git a/src/lay/lay/laySaltGrain.cc b/src/lay/lay/laySaltGrain.cc index b956e11a0..e887066e7 100644 --- a/src/lay/lay/laySaltGrain.cc +++ b/src/lay/lay/laySaltGrain.cc @@ -22,6 +22,7 @@ #include "laySaltGrain.h" #include "laySaltController.h" +#include "laySaltParsedURL.h" #include "tlString.h" #include "tlXMLParser.h" #include "tlHttpStream.h" @@ -44,7 +45,7 @@ namespace lay static const std::string grain_filename = "grain.xml"; SaltGrain::SaltGrain () - : m_protocol (DefaultProtocol), m_hidden (false) + : m_hidden (false) { // .. nothing yet .. } @@ -68,9 +69,7 @@ SaltGrain::operator== (const SaltGrain &other) const m_license == other.m_license && m_hidden == other.m_hidden && m_authored_time == other.m_authored_time && - m_installed_time == other.m_installed_time && - m_protocol == other.m_protocol && - m_branch == other.m_branch + m_installed_time == other.m_installed_time ; } @@ -116,18 +115,6 @@ SaltGrain::set_url (const std::string &u) m_url = u; } -void -SaltGrain::set_branch (const std::string &b) -{ - m_branch = b; -} - -void -SaltGrain::set_protocol (const Protocol &p) -{ - m_protocol = p; -} - void SaltGrain::set_title (const std::string &t) { @@ -410,30 +397,6 @@ struct ImageConverter } }; -struct ProtocolConverter -{ - std::string to_string (const Protocol &protocol) const - { - if (protocol == lay::WebDAV) { - return std::string ("svn"); - } else if (protocol == lay::Git) { - return std::string ("git"); - } else { - return std::string (); - } - } - - void from_string (const std::string &s, Protocol &res) const - { - res = lay::DefaultProtocol; - if (s == "svn" || s == "SVN" || s == "WebDAV") { - res = lay::WebDAV; - } else if (s == "git" || s == "Git" || s == "GIT") { - res = lay::Git; - } - } -}; - static tl::XMLElementList *sp_xml_elements = 0; tl::XMLElementList & @@ -450,8 +413,6 @@ SaltGrain::xml_elements () tl::make_member (&SaltGrain::doc, &SaltGrain::set_doc, "doc") + tl::make_member (&SaltGrain::doc_url, &SaltGrain::set_doc_url, "doc-url") + tl::make_member (&SaltGrain::url, &SaltGrain::set_url, "url") + - tl::make_member (&SaltGrain::branch, &SaltGrain::set_branch, "branch") + - tl::make_member (&SaltGrain::protocol, &SaltGrain::set_protocol, "protocol", ProtocolConverter ()) + tl::make_member (&SaltGrain::license, &SaltGrain::set_license, "license") + tl::make_member (&SaltGrain::author, &SaltGrain::set_author, "author") + tl::make_member (&SaltGrain::author_contact, &SaltGrain::set_author_contact, "author-contact") + @@ -462,8 +423,6 @@ SaltGrain::xml_elements () tl::make_element (&SaltGrain::begin_dependencies, &SaltGrain::end_dependencies, &SaltGrain::add_dependency, "depends", tl::make_member (&SaltGrainDependency::name, "name") + tl::make_member (&SaltGrainDependency::url, "url") + - tl::make_member (&SaltGrainDependency::protocol, "protocol", ProtocolConverter ()) + - tl::make_member (&SaltGrainDependency::branch, "branch") + tl::make_member (&SaltGrainDependency::version, "version") ) ); @@ -550,14 +509,17 @@ SaltGrain::from_path (const std::string &path) } tl::InputStream * -SaltGrain::stream_from_url (std::string &url, Protocol protocol, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +SaltGrain::stream_from_url (std::string &generic_url, double timeout, tl::InputHttpStreamCallback *callback) { - if (url.empty ()) { + if (generic_url.empty ()) { throw tl::Exception (tl::to_string (QObject::tr ("No download link available"))); } + lay::SaltParsedURL purl (generic_url); + const std::string &url = purl.url (); + // base relative URL's on the salt mine URL - if (url.find ("http:") != 0 && url.find ("https:") != 0 && url.find ("file:") != 0 && !url.empty() && url[0] != '/' && url[0] != '\\' && lay::SaltController::instance ()) { + if (purl.protocol () == lay::DefaultProtocol && url.find ("http:") != 0 && url.find ("https:") != 0 && url.find ("file:") != 0 && !url.empty() && url[0] != '/' && url[0] != '\\' && lay::SaltController::instance ()) { // replace the last component ("repository.xml") by the given path QUrl sami_url (tl::to_qstring (lay::SaltController::instance ()->salt_mine_url ())); @@ -567,15 +529,16 @@ SaltGrain::stream_from_url (std::string &url, Protocol protocol, const std::stri } sami_url.setPath (path_comp.join (QString::fromUtf8 ("/"))); - url = tl::to_string (sami_url.toString ()); + // return the full path as a file path, not an URL + generic_url = tl::to_string (sami_url.toString ()); } if (url.find ("http:") == 0 || url.find ("https:") == 0) { - if (protocol == lay::Git) { + if (purl.protocol () == lay::Git) { #if defined(HAVE_GIT2) - return tl::GitObject::download_item (url, SaltGrain::spec_file (), branch, timeout, callback); + return tl::GitObject::download_item (url, SaltGrain::spec_file (), purl.subfolder (), purl.branch (), timeout, callback); #else throw tl::Exception (tl::to_string (QObject::tr ("Cannot download from Git - Git support not compiled in"))); #endif @@ -591,10 +554,10 @@ SaltGrain::stream_from_url (std::string &url, Protocol protocol, const std::stri } SaltGrain -SaltGrain::from_url (const std::string &url_in, Protocol protocol, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +SaltGrain::from_url (const std::string &url_in, double timeout, tl::InputHttpStreamCallback *callback) { std::string url = url_in; - std::unique_ptr stream (stream_from_url (url, protocol, branch, timeout, callback)); + std::unique_ptr stream (stream_from_url (url, timeout, callback)); SaltGrain g; g.load (*stream); diff --git a/src/lay/lay/laySaltGrain.h b/src/lay/lay/laySaltGrain.h index 1919e98ab..58e429ffa 100644 --- a/src/lay/lay/laySaltGrain.h +++ b/src/lay/lay/laySaltGrain.h @@ -39,15 +39,6 @@ namespace tl namespace lay { -/** - * @brief An enum describing the protocol to use for download - */ -enum Protocol { - DefaultProtocol = 0, - WebDAV = 1, - Git = 2 -}; - /** * @brief A descriptor for one dependency * A dependency can be specified either through a name (see name property) @@ -59,13 +50,10 @@ enum Protocol { struct SaltGrainDependency { SaltGrainDependency () - : protocol (DefaultProtocol) { } std::string name; std::string url; - Protocol protocol; - std::string branch; std::string version; bool operator== (const SaltGrainDependency &other) const @@ -356,32 +344,6 @@ public: */ void set_url (const std::string &u); - /** - * @brief Gets the download protocol - */ - const Protocol &protocol () const - { - return m_protocol; - } - - /** - * @brief Sets the download protocol - */ - void set_protocol (const Protocol &p); - - /** - * @brief Gets the Git branch - */ - const std::string &branch () const - { - return m_branch; - } - - /** - * @brief Sets the Git branch - */ - void set_branch (const std::string &b); - /** * @brief Gets a value indicating whether the grain is hidden * A grain can be hidden (in Salt.Mine) if it's a pure dependency package @@ -507,18 +469,16 @@ public: * This method will return a grain constructed from the downloaded data. * The data is read from "URL/grain.xml". This method will throw an * exception if an error occurs during reading. - * protocol is the protocol to use and branch the Git branch. */ - static SaltGrain from_url (const std::string &url, Protocol protocol, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** * @brief Returns a stream prepared for downloading the grain * The stream is a new'd object and needs to be deleted by the caller. * "url" is the download URL on input and gets modified to match the * actual URL if it is a relative one. - * protocol is the protocol to use and branch the Git branch. */ - static tl::InputStream *stream_from_url (std::string &url, Protocol protocol, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** * @brief Gets the name of the spec file ("grain.xml") @@ -542,8 +502,6 @@ private: std::string m_author; std::string m_author_contact; std::string m_license; - Protocol m_protocol; - std::string m_branch; bool m_hidden; QDateTime m_authored_time, m_installed_time; QImage m_icon, m_screenshot; diff --git a/src/lay/lay/laySaltGrainDetailsTextWidget.cc b/src/lay/lay/laySaltGrainDetailsTextWidget.cc index 28eb81c6a..1992b20c9 100644 --- a/src/lay/lay/laySaltGrainDetailsTextWidget.cc +++ b/src/lay/lay/laySaltGrainDetailsTextWidget.cc @@ -305,6 +305,7 @@ SaltGrainDetailsTextWidget::details_text () if (! d->url.empty ()) { stream << " - "; stream << "[" << tl::to_qstring (tl::escaped_to_html (d->url)) << "]
"; + // @@@ TODO: protocol and branch } } stream << "

"; diff --git a/src/lay/lay/laySaltGrainPropertiesDialog.cc b/src/lay/lay/laySaltGrainPropertiesDialog.cc index 200bce4eb..fa6321f6e 100644 --- a/src/lay/lay/laySaltGrainPropertiesDialog.cc +++ b/src/lay/lay/laySaltGrainPropertiesDialog.cc @@ -178,6 +178,7 @@ SaltGrainPropertiesDialog::update_controls () dependency_changed (item, 0); item->setData (1, Qt::UserRole, tl::to_qstring (d->version)); dependency_changed (item, 1); + // @@@ TODO: protocol and branch? item->setData (2, Qt::UserRole, tl::to_qstring (d->url)); dependency_changed (item, 2); @@ -244,6 +245,7 @@ SaltGrainPropertiesDialog::update_data () dep.name = tl::to_string (name); dep.version = tl::to_string (version); dep.url = tl::to_string (url); + // @@@ TODO: set protocol and branch m_grain.dependencies ().push_back (dep); } @@ -584,9 +586,10 @@ SaltGrainPropertiesDialog::accept () } if (!d->url.empty ()) { + // @@@ TODO: only do for SVN repo SaltGrain gdep; try { - gdep = SaltGrain::from_url (d->url, d->protocol, d->branch); + gdep = SaltGrain::from_url (d->url); if (gdep.name () != d->name) { dependencies_alert->error () << tr ("Package name obtained from download URL is not the expected name.") << tl::endl << tr ("Downloaded name: ") << gdep.name () << tl::endl diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index 4e135e266..98f084ab7 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -683,7 +683,7 @@ BEGIN_PROTECTED SaltGrain *g = model->grain_from_index (index); // NOTE: checking for valid_name prevents bad entries inside the download list if (g && model->is_marked (g->name ()) && SaltGrain::valid_name (g->name ())) { - manager.register_download (g->name (), g->token (), g->url (), g->protocol (), g->branch (), g->version ()); + manager.register_download (g->name (), g->token (), g->url (), g->version ()); any = true; } } @@ -1190,17 +1190,13 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex details->setHtml (html); std::string url = g->url (); - Protocol protocol = g->protocol (); - std::string branch = g->branch (); m_downloaded_grain.reset (new SaltGrain ()); m_downloaded_grain->set_url (url); - m_downloaded_grain->set_protocol (protocol); - m_downloaded_grain->set_branch (branch); // NOTE: stream_from_url may modify the URL, hence we set it again ProcessEventCallback callback; - m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, protocol, branch, 60.0, &callback)); + m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); m_downloaded_grain->set_url (url); tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); diff --git a/src/lay/lay/laySaltParsedURL.h b/src/lay/lay/laySaltParsedURL.h index bc076f76b..770395a64 100644 --- a/src/lay/lay/laySaltParsedURL.h +++ b/src/lay/lay/laySaltParsedURL.h @@ -29,6 +29,15 @@ namespace lay { +/** + * @brief An enum describing the protocol to use for download + */ +enum Protocol { + DefaultProtocol = 0, + WebDAV = 1, + Git = 2 +}; + /** * @brief A class representing a SaltGrain URL * diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 3279bef16..eb2336b7f 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -243,26 +243,17 @@ checkout_branch (git_repository *repo, git_remote *remote, const git_checkout_op } void -GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { std::string url = org_url; + std::string filter = org_filter; - - std::string subdir; - - std::string url_terminator (".git/"); - size_t url_end = url.find (url_terminator); - if (url_end != std::string::npos) { - - subdir = std::string (url, url_end + url_terminator.size ()); - - url = std::string (url, 0, url_end + url_terminator.size () - 1); + if (! subfolder.empty ()) { if (filter.empty ()) { - filter = subdir + "/**"; + filter = subfolder + "/**"; } else { - filter = subdir + "/" + filter; + filter = subfolder + "/" + filter; } - } // @@@ use callback, timeout? @@ -343,9 +334,9 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons } // pull subfolder files to target path level - if (! subdir.empty ()) { + if (! subfolder.empty ()) { - std::string pp = tl::combine_path (m_local_path, subdir); + std::string pp = tl::combine_path (m_local_path, subfolder); if (! tl::is_dir (pp)) { throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to fetch subdirectory: ")) + pp); } @@ -358,7 +349,7 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons break; } } - auto pc = tl::split (subdir, "/"); + auto pc = tl::split (subfolder, "/"); if (! tl::rename_file (tl::combine_path (m_local_path, pc.front ()), tmp_dir)) { throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to rename temp folder"))); } @@ -372,18 +363,18 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons } bool -GitObject::download (const std::string &url, const std::string &target, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +GitObject::download (const std::string &url, const std::string &target, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { GitObject obj (target); - obj.read (url, std::string (), branch, timeout, callback); + obj.read (url, std::string (), subfolder, branch, timeout, callback); return false; } tl::InputStream * -GitObject::download_item (const std::string &url, const std::string &file, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +GitObject::download_item (const std::string &url, const std::string &file, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { GitObject obj; - obj.read (url, file, branch, timeout, callback); + obj.read (url, file, subfolder, branch, timeout, callback); // extract the file and return a memory blob, so we can delete the temp folder diff --git a/src/tl/tl/tlGit.h b/src/tl/tl/tlGit.h index 68a1db83a..a5d22e342 100644 --- a/src/tl/tl/tlGit.h +++ b/src/tl/tl/tlGit.h @@ -65,7 +65,7 @@ public: * "filter" can be a top-level file to download. If filter is non-empty, * sparse mode is chosen. */ - void read (const std::string &url, const std::string &filter, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + void read (const std::string &url, const std::string &filter, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** * @brief Downloads the collection or file with the given URL @@ -85,7 +85,7 @@ public: * "branch" is the remote ref to use. This can be a branch name, a tag name, * a remote ref such as "refs/heads/master" or a symbolic name such as "HEAD". */ - static bool download (const std::string &url, const std::string &target, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static bool download (const std::string &url, const std::string &target, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** * @brief Gets a stream object for downloading the single item of the given URL @@ -93,7 +93,7 @@ public: * The file needs to be a top-level object. * The stream object returned needs to be deleted by the caller. */ - static tl::InputStream *download_item (const std::string &url, const std::string &file, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static tl::InputStream *download_item (const std::string &url, const std::string &file, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); private: std::string m_local_path; diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc index cd9c5fd9e..82d62dacc 100644 --- a/src/tl/unit_tests/tlGitTests.cc +++ b/src/tl/unit_tests/tlGitTests.cc @@ -32,7 +32,7 @@ TEST(1_plain) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url, std::string (), std::string ()); + repo.read (test_url, std::string (), std::string (), std::string ()); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), true); @@ -41,11 +41,24 @@ TEST(1_plain) EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); } -TEST(2_pathspecs) +TEST(2_subdir) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url, std::string ("src/**"), std::string ()); + repo.read (test_url, std::string (), std::string ("src"), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros/xsection.lym")), true); +} + +TEST(3_subdir_as_filter) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("src/**"), std::string (), std::string ()); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); @@ -54,22 +67,11 @@ TEST(2_pathspecs) EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); } -TEST(3_subdir) -{ - std::string path = tl::TestBase::tmp_file ("repo"); - tl::GitObject repo (path); - repo.read (test_url + "/src", std::string (), std::string ()); - - EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); - EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); - EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros/xsection.lym")), true); -} - TEST(4_single_file) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url, std::string ("LICENSE"), std::string ()); + repo.read (test_url, std::string ("LICENSE"), std::string (), std::string ()); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); @@ -81,7 +83,7 @@ TEST(5_single_file_from_subdir) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url + "/src", std::string ("grain.xml"), std::string ()); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ()); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); @@ -103,7 +105,7 @@ TEST(6_branch) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("wip")); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("wip")); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); @@ -125,7 +127,7 @@ TEST(7_tag) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("1.2")); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("1.2")); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); @@ -147,7 +149,7 @@ TEST(8_refspec) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("refs/tags/1.5")); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("refs/tags/1.5")); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); @@ -169,7 +171,7 @@ TEST(9_HEAD) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("HEAD")); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("HEAD")); EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); @@ -192,7 +194,7 @@ TEST(10_invalid_branch) std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); try { - repo.read (test_url, std::string (), std::string ("brxxx")); + repo.read (test_url, std::string (), std::string (), std::string ("brxxx")); EXPECT_EQ (true, false); } catch (tl::Exception &ex) { EXPECT_EQ (ex.msg (), "Git checkout - Unable to resolve reference name: brxxx"); From e83b3c1477601d61d1c2fdf455aef1e70e3ccfae Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 29 Oct 2023 17:37:11 +0100 Subject: [PATCH 12/26] Trying to avoid package downloads when possible, taking the information from the index --- src/lay/lay/laySaltDownloadManager.cc | 72 ++++++++++++---- src/lay/lay/laySaltGrain.h | 4 + src/lay/lay/laySaltGrainPropertiesDialog.cc | 12 ++- src/lay/lay/laySaltManagerDialog.cc | 93 +++++++++++++-------- 4 files changed, 128 insertions(+), 53 deletions(-) diff --git a/src/lay/lay/laySaltDownloadManager.cc b/src/lay/lay/laySaltDownloadManager.cc index cc0b625d7..ea0202810 100644 --- a/src/lay/lay/laySaltDownloadManager.cc +++ b/src/lay/lay/laySaltDownloadManager.cc @@ -27,6 +27,7 @@ #include "tlFileUtils.h" #include "tlWebDAV.h" #include "tlLog.h" +#include "tlEnv.h" #include #include @@ -38,6 +39,14 @@ namespace lay // ---------------------------------------------------------------------------------- +static bool download_package_information () +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info"); +} + +// ---------------------------------------------------------------------------------- + ConfirmationDialog::ConfirmationDialog (QWidget *parent) : QDialog (parent), m_confirmed (false), m_cancelled (false), m_aborted (false), m_file (50000, true) { @@ -310,43 +319,78 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt ++progress; // Add URL and token from the package index + // + // In order to do so, we try to use the information from that package index as far as possible. + // Downloading a package definition from the original package URL may be expensive in case of + // large GIT repositories. + // + // Downloading is required if: + // - A package download is requested without a name (package can't be looked up in the package index) + // - Or a name is given, but not found in the package index + if (! p->name.empty ()) { const lay::SaltGrain *g = salt_mine.grain_by_name (p->name); if (! g) { if (p->url.empty ()) { - throw tl::Exception (tl::to_string (QObject::tr ("Package '%1' not found in index - cannot resolve download URL").arg (tl::to_qstring (p->name)))); + throw tl::Exception (tl::to_string (tr ("Package '%s' not found in index - cannot resolve download URL")), p->name); } } else { if (p->url.empty ()) { if (tl::verbosity() >= 20) { - tl::log << "Resolved package URL for package " << p->name << ": " << g->url (); + tl::log << tr ("Resolved package URL for package") << " '" << p->name << "': " << g->url (); } p->url = g->url (); } p->token = g->token (); + p->grain = *g; + p->downloaded = true; } } - try { - // @@@ Take from repo index for Git protocol? - p->grain = SaltGrain::from_url (p->url); - } catch (tl::Exception &ex) { - throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ())))); + if (! p->downloaded && download_package_information ()) { + + // If requested, download package information to complete information from index or dependencies + if (tl::verbosity() >= 10) { + tl::log << tl::sprintf (tl::to_string (tr ("Reading package description for package '%s' from: '%s")), p->name, p->url); + } + try { + p->grain = SaltGrain::from_url (p->url); + p->downloaded = true; + } catch (tl::Exception &ex) { + throw tl::Exception (tl::to_string (tr ("Error fetching spec file for package from '%s': %s")), p->url, ex.msg ()); + } + } - if (p->version.empty ()) { - p->version = p->grain.version (); + if (! p->downloaded) { + + if (p->name.empty ()) { + throw tl::Exception (tl::to_string (tr ("No name given package from '%s' (from dependencies or command line installation request)")), p->url); + } + + if (tl::verbosity() >= 10) { + tl::warn << tl::sprintf (tl::to_string (tr ("Package '%s' not downloaded from: %s. Dependencies may not be resolved.")), p->name, p->url); + } + + } else { + + if (p->version.empty ()) { + p->version = p->grain.version (); + } + if (p->name.empty ()) { + p->name = p->grain.name (); + } + + if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) { + throw tl::Exception (tl::to_string (tr ("Package '%s': package in repository is too old (%s) to satisfy requirements (%s)")), p->name, p->grain.version (), p->version); + } + } - p->name = p->grain.name (); p->downloaded = true; - if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) { - throw tl::Exception (tl::to_string (QObject::tr ("Package '%1': package in repository is too old (%2) to satisfy requirements (%3)").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (p->grain.version ())).arg (tl::to_qstring (p->version)))); - } - } } diff --git a/src/lay/lay/laySaltGrain.h b/src/lay/lay/laySaltGrain.h index 58e429ffa..bf0f2ef90 100644 --- a/src/lay/lay/laySaltGrain.h +++ b/src/lay/lay/laySaltGrain.h @@ -469,6 +469,8 @@ public: * This method will return a grain constructed from the downloaded data. * The data is read from "URL/grain.xml". This method will throw an * exception if an error occurs during reading. + * + * CAUTION: with GIT protocol and large repositories, this function may be very expensive. */ static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); @@ -477,6 +479,8 @@ public: * The stream is a new'd object and needs to be deleted by the caller. * "url" is the download URL on input and gets modified to match the * actual URL if it is a relative one. + * + * CAUTION: with GIT protocol and large repositories, this function may be very expensive. */ static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); diff --git a/src/lay/lay/laySaltGrainPropertiesDialog.cc b/src/lay/lay/laySaltGrainPropertiesDialog.cc index fa6321f6e..22ad35005 100644 --- a/src/lay/lay/laySaltGrainPropertiesDialog.cc +++ b/src/lay/lay/laySaltGrainPropertiesDialog.cc @@ -25,6 +25,7 @@ #include "tlString.h" #include "tlExceptions.h" #include "tlHttpStream.h" +#include "tlEnv.h" #include #include @@ -43,6 +44,14 @@ namespace lay // ---------------------------------------------------------------------------------------------------- +static bool download_package_information () +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info"); +} + +// ---------------------------------------------------------------------------------------------------- + /** * @brief A delegate for editing a field of the dependency list */ @@ -585,8 +594,7 @@ SaltGrainPropertiesDialog::accept () << tr ("If the dependency package has a version itself, the version is automatically set to its current version."); } - if (!d->url.empty ()) { - // @@@ TODO: only do for SVN repo + if (!d->url.empty () && download_package_information ()) { SaltGrain gdep; try { gdep = SaltGrain::from_url (d->url); diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index 98f084ab7..de2531de8 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -30,6 +30,7 @@ #include "ui_SaltGrainTemplateSelectionDialog.h" #include "tlString.h" #include "tlExceptions.h" +#include "tlEnv.h" #include "rba.h" #include "pya.h" @@ -51,6 +52,14 @@ namespace lay // -------------------------------------------------------------------------------------- +static bool download_package_information () +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info"); +} + +// -------------------------------------------------------------------------------------- + /** * @brief A tiny dialog to select a template and a name for the grain */ @@ -1168,48 +1177,58 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex mp_downloaded_target = details; m_salt_mine_grain.reset (new lay::SaltGrain (*g)); - // Download actual grain definition file - try { + if (download_package_information ()) { - if (g->url ().empty ()) { - throw tl::Exception (tl::to_string (tr ("No download link available"))); + // Download actual grain definition file + try { + + if (g->url ().empty ()) { + throw tl::Exception (tl::to_string (tr ("No download link available"))); + } + + QString html = tr ( + "" + "" + "" + "

Fetching Package Definition ...

" + "

URL: %1

" + "" + "" + "" + ) + .arg (tl::to_qstring (g->url ())); + + details->setHtml (html); + + std::string url = g->url (); + + m_downloaded_grain.reset (new SaltGrain ()); + m_downloaded_grain->set_url (url); + + // NOTE: stream_from_url may modify the URL, hence we set it again + ProcessEventCallback callback; + m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); + m_downloaded_grain->set_url (url); + + tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); + if (http) { + // async reading on HTTP + http->ready ().add (this, &SaltManagerDialog::data_ready); + http->send (); + } else { + data_ready (); + } + + } catch (tl::Exception &ex) { + show_error (ex); } - QString html = tr ( - "" - "" - "" - "

Fetching Package Definition ...

" - "

URL: %1

" - "" - "" - "" - ) - .arg (tl::to_qstring (g->url ())); + } else { - details->setHtml (html); + // Download denied - take information from index + m_downloaded_grain.reset (new SaltGrain (*g)); + data_ready (); - std::string url = g->url (); - - m_downloaded_grain.reset (new SaltGrain ()); - m_downloaded_grain->set_url (url); - - // NOTE: stream_from_url may modify the URL, hence we set it again - ProcessEventCallback callback; - m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); - m_downloaded_grain->set_url (url); - - tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); - if (http) { - // async reading on HTTP - http->ready ().add (this, &SaltManagerDialog::data_ready); - http->send (); - } else { - data_ready (); - } - - } catch (tl::Exception &ex) { - show_error (ex); } } From 3e34d205e8625bd9ad55218e067c48334197406e Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 29 Oct 2023 20:24:28 +0100 Subject: [PATCH 13/26] Fixed Git parsed URL scheme to align to SVN emulation of GitHub --- src/lay/lay/laySaltParsedURL.cc | 18 ++++++++---------- src/lay/lay/laySaltParsedURL.h | 4 ++-- src/lay/unit_tests/laySaltParsedURLTests.cc | 4 ++-- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/lay/lay/laySaltParsedURL.cc b/src/lay/lay/laySaltParsedURL.cc index 8df57de5f..11767b1b4 100644 --- a/src/lay/lay/laySaltParsedURL.cc +++ b/src/lay/lay/laySaltParsedURL.cc @@ -101,24 +101,22 @@ parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::st // SVN emulation auto parts = tl::split (subfolder, "/"); - if (parts.size () >= 1 && parts.back () == "trunk") { + if (parts.size () >= 1 && parts.front () == "trunk") { branch = "HEAD"; - parts.pop_back (); + parts.erase (parts.begin ()); subfolder = tl::join (parts, "/"); - } else if (parts.size () >= 2 && parts[parts.size () - 2] == "tags") { + } else if (parts.size () >= 2 && parts.front () == "tags") { - branch = "refs/tags/" + parts.back (); - parts.pop_back (); - parts.pop_back (); + branch = "refs/tags/" + parts[1]; + parts.erase (parts.begin (), parts.begin () + 2); subfolder = tl::join (parts, "/"); - } else if (parts.size () >= 2 && parts[parts.size () - 2] == "branches") { + } else if (parts.size () >= 2 && parts.front () == "branches") { - branch = "refs/heads/" + parts.back (); - parts.pop_back (); - parts.pop_back (); + branch = "refs/heads/" + parts[1]; + parts.erase (parts.begin (), parts.begin () + 2); subfolder = tl::join (parts, "/"); } diff --git a/src/lay/lay/laySaltParsedURL.h b/src/lay/lay/laySaltParsedURL.h index 770395a64..fd41bb42d 100644 --- a/src/lay/lay/laySaltParsedURL.h +++ b/src/lay/lay/laySaltParsedURL.h @@ -55,10 +55,10 @@ enum Protocol { * git+https://server.com/repo.git[v1.0] -> protocol=Git, url="https://server.com/repo.git", branch="v1.0", subfolder="" * git+https://server.com/repo.git/sub/folder[refs/tags/1.0] -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.0", subfolder="sub/folder" * git+https://server.com/repo.git/trunk -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="" - * git+https://server.com/repo.git/sub/folder/trunk -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="sub/folder" + * git+https://server.com/repo.git/trunk/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="sub/folder" * git+https://server.com/repo.git/branches/release -> protocol=Git, url="https://server.com/repo.git", branch="refs/heads/release", subfolder="" * git+https://server.com/repo.git/tags/1.9 -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="" - * git+https://server.com/repo.git/sub/folder/tags/1.9 -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="sub/folder" + * git+https://server.com/repo.git/tags/1.9/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="sub/folder" */ class LAY_PUBLIC SaltParsedURL diff --git a/src/lay/unit_tests/laySaltParsedURLTests.cc b/src/lay/unit_tests/laySaltParsedURLTests.cc index 3b4985d88..96e22e5b6 100644 --- a/src/lay/unit_tests/laySaltParsedURLTests.cc +++ b/src/lay/unit_tests/laySaltParsedURLTests.cc @@ -97,7 +97,7 @@ TEST (15_GitSVNEmulationTrunk) TEST (16_GitSVNEmulationTrunkWithSubFolder) { - lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder/trunk"); + lay::SaltParsedURL purl ("git+https://server.com/repo.git/trunk/sub/folder"); EXPECT_EQ (purl.protocol () == lay::Git, true); EXPECT_EQ (purl.url (), "https://server.com/repo.git"); EXPECT_EQ (purl.branch (), "HEAD"); @@ -124,7 +124,7 @@ TEST (18_GitSVNEmulationTag) TEST (19_GitSVNEmulationTagWithSubFolder) { - lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder/tags/1.9"); + lay::SaltParsedURL purl ("git+https://server.com/repo.git/tags/1.9/sub/folder"); EXPECT_EQ (purl.protocol () == lay::Git, true); EXPECT_EQ (purl.url (), "https://server.com/repo.git"); EXPECT_EQ (purl.branch (), "refs/tags/1.9"); From a4df1eb10fff55efa387583e0b230e324743c8c0 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 29 Oct 2023 21:44:31 +0100 Subject: [PATCH 14/26] Some bug fixing --- src/lay/lay/laySaltGrain.cc | 4 ++++ src/lay/lay/laySaltParsedURL.cc | 7 +++++-- src/lay/unit_tests/laySaltParsedURLTests.cc | 9 +++++++++ src/tl/tl/tlGit.cc | 18 ++++++++++++------ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/lay/lay/laySaltGrain.cc b/src/lay/lay/laySaltGrain.cc index e887066e7..cd54e841f 100644 --- a/src/lay/lay/laySaltGrain.cc +++ b/src/lay/lay/laySaltGrain.cc @@ -515,6 +515,10 @@ SaltGrain::stream_from_url (std::string &generic_url, double timeout, tl::InputH throw tl::Exception (tl::to_string (QObject::tr ("No download link available"))); } + if (tl::verbosity () >= 20) { + tl::info << tr ("Downloading package info from ") << generic_url; + } + lay::SaltParsedURL purl (generic_url); const std::string &url = purl.url (); diff --git a/src/lay/lay/laySaltParsedURL.cc b/src/lay/lay/laySaltParsedURL.cc index 11767b1b4..84e2c0850 100644 --- a/src/lay/lay/laySaltParsedURL.cc +++ b/src/lay/lay/laySaltParsedURL.cc @@ -39,7 +39,7 @@ parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::st ; } // server ("www.klayout.de") - while (! ex.at_end () && (*ex != '/' && *ex != '+')) { + while (! ex.at_end () && (*ex != '/' && *ex != '+' && *ex != '[')) { ++ex; } @@ -47,6 +47,7 @@ parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::st ++ex; + // next component const char *c1 = ex.get (); while (! ex.at_end () && (*ex != '/' && *ex != '+' && *ex != '[')) { ++ex; @@ -55,7 +56,7 @@ parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::st std::string comp (c1, c2 - c1); - if ((! ex.at_end () && *ex == '+') || comp.find (".git") == comp.size () - 4) { + if ((! ex.at_end () && (*ex == '+' || *ex == '[')) || comp.find (".git") == comp.size () - 4) { // subfolder starts here break; } @@ -68,6 +69,7 @@ parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::st return; } + // skip URL/subfolder separator if (*ex == '/') { while (! ex.at_end () && *ex == '/') { ++ex; @@ -76,6 +78,7 @@ parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::st ++ex; } + // subfolders { const char *c1 = ex.get (); while (! ex.at_end () && *ex != '[') { diff --git a/src/lay/unit_tests/laySaltParsedURLTests.cc b/src/lay/unit_tests/laySaltParsedURLTests.cc index 96e22e5b6..b4274b3fd 100644 --- a/src/lay/unit_tests/laySaltParsedURLTests.cc +++ b/src/lay/unit_tests/laySaltParsedURLTests.cc @@ -130,3 +130,12 @@ TEST (19_GitSVNEmulationTagWithSubFolder) EXPECT_EQ (purl.branch (), "refs/tags/1.9"); EXPECT_EQ (purl.subfolder (), "sub/folder"); } + +TEST (20_Example1) +{ + lay::SaltParsedURL purl ("git+https://github.com/my-user/test-core[refs/tags/v1.1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://github.com/my-user/test-core"); + EXPECT_EQ (purl.branch (), "refs/tags/v1.1.0"); + EXPECT_EQ (purl.subfolder (), ""); +} diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index eb2336b7f..d1e100cad 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -79,17 +79,23 @@ GitObject::GitObject (const std::string &local_path) if (local_path.empty ()) { // @@@ generic tempnam on Windows/Posix ... - char tmp[] = "git2klayoutXXXXXX"; - mkstemp (tmp); + char tmp[] = "/tmp/git2klayoutXXXXXX"; + if (mkdtemp (tmp) == NULL) { + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder: %s")), (const char *) tmp); + } m_local_path = tmp; m_is_temp = true; } - tl::mkpath (m_local_path); - // ensures the directory is clean - tl::rm_dir_recursive (m_local_path); // @@@ TODO: error handling? - tl::mkpath (m_local_path); + if (! m_is_temp) { + if (! tl::rm_dir_recursive (m_local_path)) { + throw tl::Exception (tl::to_string (tr ("Unable to clean local Git repo path: %s")), m_local_path); + } + if (! tl::mkpath (m_local_path)) { + throw tl::Exception (tl::to_string (tr ("Unable to regenerate local Git repo path: %s")), m_local_path); + } + } } GitObject::~GitObject () From 9b969c25bedab0edb52e4f437255803a61cf50af Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 29 Oct 2023 22:22:53 +0100 Subject: [PATCH 15/26] Enabling progress for Git checkout --- src/lay/lay/laySaltController.cc | 6 +++--- src/tl/tl/tlFileSystemWatcher.h | 17 +++++++++++++++++ src/tl/tl/tlGit.cc | 12 ++++++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/lay/lay/laySaltController.cc b/src/lay/lay/laySaltController.cc index efd85a58e..51b53ae66 100644 --- a/src/lay/lay/laySaltController.cc +++ b/src/lay/lay/laySaltController.cc @@ -143,7 +143,7 @@ SaltController::show_editor () { // while running the dialog, don't watch file events - that would interfere with // the changes applied by the dialog itself. - lay::BusySection busy_section; // disable file watcher + tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher mp_salt_dialog->exec (); } @@ -157,7 +157,7 @@ SaltController::show_editor () void SaltController::sync_file_watcher () { - lay::BusySection busy_section; // disable file watcher + tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher if (m_file_watcher) { m_file_watcher->clear (); @@ -231,7 +231,7 @@ SaltController::install_packages (const std::vector &packages, bool { // while running the dialog, don't watch file events - that would interfere with // the changes applied by the dialog itself. - lay::BusySection busy_section; // disable file watcher + tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher result = manager.execute (0, m_salt); } diff --git a/src/tl/tl/tlFileSystemWatcher.h b/src/tl/tl/tlFileSystemWatcher.h index f6190f208..d7435c261 100644 --- a/src/tl/tl/tlFileSystemWatcher.h +++ b/src/tl/tl/tlFileSystemWatcher.h @@ -140,6 +140,23 @@ private: std::map::iterator m_iter; }; +/** + * @brief A class employing RIIA for locking the file system watcher + */ +class TL_PUBLIC FileSystemWatcherDisabled +{ +public: + FileSystemWatcherDisabled () + { + tl::FileSystemWatcher::global_enable (false); + } + + ~FileSystemWatcherDisabled () + { + tl::FileSystemWatcher::global_enable (true); + } +}; + } #endif diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index d1e100cad..2c8ee8b6a 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -116,7 +116,11 @@ fetch_progress (const git_transfer_progress *stats, void *payload) // first half of progress size_t count = size_t (5000.0 * double (stats->received_objects) / double (std::max (1u, stats->total_objects)) + 1e-10); - progress->set (count); + try { + progress->set (count); + } catch (...) { + // TODO: stop + } return 0; } @@ -128,7 +132,11 @@ checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload) // first half of progress size_t count = size_t (5000.0 * double (cur) / double (std::max (size_t (1), tot)) + 1e-10); - progress->set (count + 5000u); + try { + progress->set (count + 5000u); + } catch (...) { + // ignore cancel requests (TODO: how to stop?) + } } static void check (int error) From 82a0ef77916be46ae75cf0d6b1c1959c5ce01515 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Tue, 31 Oct 2023 22:05:29 +0100 Subject: [PATCH 16/26] First git-enabled package manager dialog --- src/lay/lay/laySaltManagerDialog.cc | 77 ++++++++++++++++++++++++++--- src/lay/lay/laySaltManagerDialog.h | 4 ++ src/tl/tl/tlGit.cc | 2 +- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index de2531de8..b034f93e8 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -299,7 +299,9 @@ SaltManagerDialog::SaltManagerDialog (QWidget *parent, lay::Salt *salt, const st : QDialog (parent), m_salt_mine_url (salt_mine_url), dm_update_models (this, &SaltManagerDialog::update_models), m_current_tab (-1), - mp_downloaded_target (0) + mp_downloaded_target (0), + dm_mine_update_selected_changed (this, &SaltManagerDialog::do_mine_update_selected_changed), + dm_mine_new_selected_changed (this, &SaltManagerDialog::do_mine_new_selected_changed) { Ui::SaltManagerDialog::setupUi (this); mp_properties_dialog = new lay::SaltGrainPropertiesDialog (this); @@ -1104,8 +1106,12 @@ SaltManagerDialog::current_grains () void SaltManagerDialog::mine_update_selected_changed () { -BEGIN_PROTECTED + dm_mine_update_selected_changed (); +} +void +SaltManagerDialog::do_mine_update_selected_changed () +{ SaltModel *model = dynamic_cast (salt_mine_view_update->model ()); tl_assert (model != 0); @@ -1118,15 +1124,17 @@ BEGIN_PROTECTED details_update_frame->setEnabled (g != 0); get_remote_grain_info (g, details_update_text); - -END_PROTECTED } void SaltManagerDialog::mine_new_selected_changed () { -BEGIN_PROTECTED + dm_mine_new_selected_changed (); +} +void +SaltManagerDialog::do_mine_new_selected_changed () +{ SaltModel *model = dynamic_cast (salt_mine_view_new->model ()); tl_assert (model != 0); @@ -1139,8 +1147,6 @@ BEGIN_PROTECTED details_new_frame->setEnabled (g != 0); get_remote_grain_info (g, details_new_text); - -END_PROTECTED } namespace @@ -1159,6 +1165,55 @@ public: } }; +class FetchGrainInfoProgressAdaptor + : public tl::ProgressAdaptor +{ +public: + FetchGrainInfoProgressAdaptor (SaltGrainDetailsTextWidget *details, const std::string &name, const QString &html) + : mp_details (details), m_name (name), m_html (html) + { + mp_details->setHtml (m_html.arg (QString ())); + m_counter = 0; + } + + virtual void yield (tl::Progress *progress) + { + QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents, 100); + + ++m_counter; + std::string all_dots = ".........."; + m_counter = m_counter % all_dots.size (); + std::string dots = std::string (all_dots, 0, m_counter); + mp_details->setHtml (m_html.arg (tl::to_qstring (tl::sprintf (tl::to_string (tr ("Downloading %.0f%% %s")), progress->value (), dots)))); + } + + virtual void trigger (tl::Progress * /*progress*/) + { + // .. nothing yet .. + } + + void error () + { + mp_details->setHtml (m_html.arg (QString ())); + } + + void success () + { + mp_details->setHtml (m_html.arg (QString ())); + } + + bool is_aborted () const + { + return false; + } + +private: + lay::SaltGrainDetailsTextWidget *mp_details; + std::string m_name; + QString m_html; + unsigned int m_counter; +}; + } void @@ -1169,6 +1224,8 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex return; } + tl_assert (m_downloaded_grain.get () == 0); + m_downloaded_grain.reset (0); if (m_downloaded_grain_reader.get ()) { // NOTE: don't delete the reader in the slot it triggered @@ -1192,13 +1249,16 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex "" "

Fetching Package Definition ...

" "

URL: %1

" + "

%2

" "" "" "" ) .arg (tl::to_qstring (g->url ())); - details->setHtml (html); + details->setHtml (html.arg (QString ())); + + FetchGrainInfoProgressAdaptor pa (details, g->name (), html); std::string url = g->url (); @@ -1263,6 +1323,7 @@ SaltManagerDialog::data_ready () m_salt_mine_grain.reset (0); } catch (tl::Exception &ex) { + m_downloaded_grain.reset (0); show_error (ex); } } diff --git a/src/lay/lay/laySaltManagerDialog.h b/src/lay/lay/laySaltManagerDialog.h index a16662cd9..db3851801 100644 --- a/src/lay/lay/laySaltManagerDialog.h +++ b/src/lay/lay/laySaltManagerDialog.h @@ -188,6 +188,8 @@ private: std::unique_ptr m_downloaded_grain, m_salt_mine_grain; SaltGrainDetailsTextWidget *mp_downloaded_target; std::unique_ptr m_salt_mine_reader; + tl::DeferredMethod dm_mine_update_selected_changed; + tl::DeferredMethod dm_mine_new_selected_changed; SaltGrain *current_grain (); std::vector current_grains (); @@ -199,6 +201,8 @@ private: void show_error (tl::Exception &ex); void salt_mine_download_started (); void salt_mine_download_finished (); + void do_mine_update_selected_changed (); + void do_mine_new_selected_changed (); }; } diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 2c8ee8b6a..95470580f 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -270,7 +270,7 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons } } - // @@@ use callback, timeout? + // TODO: use callback, timeout? tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/); // build checkout options From 579bee3b75afbcf1d70a13633aaf00e74a592880 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Tue, 31 Oct 2023 22:30:52 +0100 Subject: [PATCH 17/26] Implemented package information cache in package manager --- src/lay/lay/laySaltManagerDialog.cc | 51 ++++++++++++++++++++--------- src/lay/lay/laySaltManagerDialog.h | 1 + 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index b034f93e8..d78aa5ebe 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -857,6 +857,8 @@ SaltManagerDialog::salt_mine_about_to_change () void SaltManagerDialog::refresh () { + m_salt_grain_cache.clear (); + if (! m_salt_mine_url.empty ()) { tl::log << tl::to_string (tr ("Downloading package repository from %1").arg (tl::to_qstring (m_salt_mine_url))); @@ -1227,10 +1229,12 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex tl_assert (m_downloaded_grain.get () == 0); m_downloaded_grain.reset (0); + if (m_downloaded_grain_reader.get ()) { - // NOTE: don't delete the reader in the slot it triggered m_downloaded_grain_reader->close (); } + m_downloaded_grain_reader.reset (0); + mp_downloaded_target = details; m_salt_mine_grain.reset (new lay::SaltGrain (*g)); @@ -1262,21 +1266,31 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex std::string url = g->url (); - m_downloaded_grain.reset (new SaltGrain ()); - m_downloaded_grain->set_url (url); + auto sg = m_salt_grain_cache.find (url); + if (sg == m_salt_grain_cache.end ()) { - // NOTE: stream_from_url may modify the URL, hence we set it again - ProcessEventCallback callback; - m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); - m_downloaded_grain->set_url (url); + m_downloaded_grain.reset (new SaltGrain ()); + m_downloaded_grain->set_url (url); + + // NOTE: stream_from_url may modify the URL, hence we set it again + ProcessEventCallback callback; + m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); + m_downloaded_grain->set_url (url); + + tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); + if (http) { + // async reading on HTTP + http->ready ().add (this, &SaltManagerDialog::data_ready); + http->send (); + } else { + data_ready (); + } - tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); - if (http) { - // async reading on HTTP - http->ready ().add (this, &SaltManagerDialog::data_ready); - http->send (); } else { + + m_downloaded_grain.reset (new SaltGrain (sg->second)); data_ready (); + } } catch (tl::Exception &ex) { @@ -1295,14 +1309,21 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex void SaltManagerDialog::data_ready () { - if (! m_salt_mine_grain.get () || ! m_downloaded_grain.get () || ! m_downloaded_grain_reader.get () || ! mp_downloaded_target) { + if (! m_salt_mine_grain.get () || ! m_downloaded_grain.get () || ! mp_downloaded_target) { return; } // Load the grain file (save URL as it is overwritten by the grain.xml content) std::string url = m_downloaded_grain->url (); - m_downloaded_grain->load (*m_downloaded_grain_reader); - m_downloaded_grain->set_url (url); + if (m_downloaded_grain_reader.get ()) { + m_downloaded_grain->load (*m_downloaded_grain_reader); + m_downloaded_grain->set_url (url); + } + + // commit to cache + if (m_salt_grain_cache.find (url) == m_salt_grain_cache.end ()) { + m_salt_grain_cache [url] = *m_downloaded_grain; + } try { diff --git a/src/lay/lay/laySaltManagerDialog.h b/src/lay/lay/laySaltManagerDialog.h index db3851801..dd899371b 100644 --- a/src/lay/lay/laySaltManagerDialog.h +++ b/src/lay/lay/laySaltManagerDialog.h @@ -190,6 +190,7 @@ private: std::unique_ptr m_salt_mine_reader; tl::DeferredMethod dm_mine_update_selected_changed; tl::DeferredMethod dm_mine_new_selected_changed; + std::map m_salt_grain_cache; SaltGrain *current_grain (); std::vector current_grains (); From 880c8fbb05499964525c7c42912dd6f5bf6810ba Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Tue, 31 Oct 2023 23:31:22 +0100 Subject: [PATCH 18/26] Introducing sparse tag for salt mine repo to indicate it is not required to always download package information --- src/lay/lay/laySalt.cc | 8 ++++++++ src/lay/lay/laySalt.h | 5 +++++ src/lay/lay/laySaltDownloadManager.cc | 13 ++++--------- src/lay/lay/laySaltGrains.cc | 11 +++++++++++ src/lay/lay/laySaltGrains.h | 19 +++++++++++++++++++ src/lay/lay/laySaltManagerDialog.cc | 10 +--------- 6 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/lay/lay/laySalt.cc b/src/lay/lay/laySalt.cc index 2abf3c519..81267fa97 100644 --- a/src/lay/lay/laySalt.cc +++ b/src/lay/lay/laySalt.cc @@ -28,6 +28,7 @@ #include "tlLog.h" #include "tlInternational.h" #include "tlWebDAV.h" +#include "tlEnv.h" #if defined(HAVE_GIT2) # include "tlGit.h" #endif @@ -69,6 +70,13 @@ Salt::root () return m_root; } +bool +Salt::download_package_information () const +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info") || m_root.sparse (); +} + Salt::flat_iterator Salt::begin_flat () { diff --git a/src/lay/lay/laySalt.h b/src/lay/lay/laySalt.h index 7970862a3..71c5a69d0 100644 --- a/src/lay/lay/laySalt.h +++ b/src/lay/lay/laySalt.h @@ -199,6 +199,11 @@ public: */ SaltGrains &root (); + /** + * @brief Gets a value indicating whether the collection wants package information to be downloaded always + */ + bool download_package_information () const; + signals: /** * @brief A signal triggered before one of the collections changed diff --git a/src/lay/lay/laySaltDownloadManager.cc b/src/lay/lay/laySaltDownloadManager.cc index ea0202810..559b59333 100644 --- a/src/lay/lay/laySaltDownloadManager.cc +++ b/src/lay/lay/laySaltDownloadManager.cc @@ -39,14 +39,6 @@ namespace lay // ---------------------------------------------------------------------------------- -static bool download_package_information () -{ - // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO - return tl::app_flag ("always-download-package-info"); -} - -// ---------------------------------------------------------------------------------- - ConfirmationDialog::ConfirmationDialog (QWidget *parent) : QDialog (parent), m_confirmed (false), m_cancelled (false), m_aborted (false), m_file (50000, true) { @@ -327,6 +319,9 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt // Downloading is required if: // - A package download is requested without a name (package can't be looked up in the package index) // - Or a name is given, but not found in the package index + // + // Downloading can be bypassed if the package index (salt mine) specifies "sparse=false". + // In that case, the package index will have all information about the package. if (! p->name.empty ()) { @@ -349,7 +344,7 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt } - if (! p->downloaded && download_package_information ()) { + if (! p->downloaded && salt_mine.download_package_information ()) { // If requested, download package information to complete information from index or dependencies if (tl::verbosity() >= 10) { diff --git a/src/lay/lay/laySaltGrains.cc b/src/lay/lay/laySaltGrains.cc index bacb19d5f..846d8052c 100644 --- a/src/lay/lay/laySaltGrains.cc +++ b/src/lay/lay/laySaltGrains.cc @@ -34,6 +34,7 @@ namespace lay { SaltGrains::SaltGrains () + : m_sparse (true) { // .. nothing yet .. } @@ -54,6 +55,12 @@ SaltGrains::set_name (const std::string &n) m_name = n; } +void +SaltGrains::set_sparse (const bool &f) +{ + m_sparse = f; +} + void SaltGrains::set_title (const std::string &t) { @@ -302,6 +309,7 @@ SaltGrains::consolidate () static tl::XMLElementList s_group_struct = tl::make_member (&SaltGrains::name, &SaltGrains::set_name, "name") + + tl::make_member (&SaltGrains::sparse, &SaltGrains::set_sparse, "sparse") + tl::make_member (&SaltGrains::include, "include") + tl::make_element (&SaltGrains::begin_collections, &SaltGrains::end_collections, &SaltGrains::add_collection, "group", &s_group_struct) + tl::make_element (&SaltGrains::begin_grains, &SaltGrains::end_grains, &SaltGrains::add_grain, "salt-grain", SaltGrain::xml_elements ()); @@ -354,6 +362,9 @@ SaltGrains::include (const std::string &src_in) lay::SaltGrains g; g.load (src); + if (g.sparse ()) { + m_sparse = true; + } m_collections.splice (m_collections.end (), g.m_collections); m_grains.splice (m_grains.end (), g.m_grains); diff --git a/src/lay/lay/laySaltGrains.h b/src/lay/lay/laySaltGrains.h index a277e7863..c43e5fbaf 100644 --- a/src/lay/lay/laySaltGrains.h +++ b/src/lay/lay/laySaltGrains.h @@ -78,6 +78,24 @@ public: */ void set_name (const std::string &p); + /** + * @brief Gets a value indicating that the information in the grain collection is sparse + * + * If this flag is set to true (the default), the information in the collection needs + * to be completed by pulling the original definition of the grain for the grain's URL. + * If the flag is false, the information is complete and reflects the grain's original + * definition. + */ + const bool &sparse () const + { + return m_sparse; + } + + /** + * @brief Sets a value indicating that the information in the grain collection is sparse + */ + void set_sparse (const bool &f); + /** * @brief Gets the title of the grain collection * @@ -225,6 +243,7 @@ private: collections_type m_collections; grains_type m_grains; std::string m_url; + bool m_sparse; }; } diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index d78aa5ebe..4e86323bd 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -52,14 +52,6 @@ namespace lay // -------------------------------------------------------------------------------------- -static bool download_package_information () -{ - // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO - return tl::app_flag ("always-download-package-info"); -} - -// -------------------------------------------------------------------------------------- - /** * @brief A tiny dialog to select a template and a name for the grain */ @@ -1238,7 +1230,7 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex mp_downloaded_target = details; m_salt_mine_grain.reset (new lay::SaltGrain (*g)); - if (download_package_information ()) { + if (m_salt_mine.download_package_information () && m_salt_mine.grain_by_name (g->name ())) { // Download actual grain definition file try { From b78f01387f0060e4c2692c494ce6e1354f40e651 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 1 Nov 2023 18:18:26 +0100 Subject: [PATCH 19/26] Generalization of temporary file and directory creation --- src/lay/lay/laySaltGrainDetailsTextWidget.cc | 1 - src/lay/lay/laySaltGrainPropertiesDialog.cc | 2 - src/tl/tl/tlFileUtils.cc | 105 +++++++++++++++++++ src/tl/tl/tlFileUtils.h | 62 +++++++++++ src/tl/tl/tlGit.cc | 7 +- src/tl/unit_tests/tlFileUtilsTests.cc | 80 ++++++++++++++ 6 files changed, 248 insertions(+), 9 deletions(-) diff --git a/src/lay/lay/laySaltGrainDetailsTextWidget.cc b/src/lay/lay/laySaltGrainDetailsTextWidget.cc index 1992b20c9..28eb81c6a 100644 --- a/src/lay/lay/laySaltGrainDetailsTextWidget.cc +++ b/src/lay/lay/laySaltGrainDetailsTextWidget.cc @@ -305,7 +305,6 @@ SaltGrainDetailsTextWidget::details_text () if (! d->url.empty ()) { stream << " - "; stream << "[" << tl::to_qstring (tl::escaped_to_html (d->url)) << "]
"; - // @@@ TODO: protocol and branch } } stream << "

"; diff --git a/src/lay/lay/laySaltGrainPropertiesDialog.cc b/src/lay/lay/laySaltGrainPropertiesDialog.cc index 22ad35005..061eee1ca 100644 --- a/src/lay/lay/laySaltGrainPropertiesDialog.cc +++ b/src/lay/lay/laySaltGrainPropertiesDialog.cc @@ -187,7 +187,6 @@ SaltGrainPropertiesDialog::update_controls () dependency_changed (item, 0); item->setData (1, Qt::UserRole, tl::to_qstring (d->version)); dependency_changed (item, 1); - // @@@ TODO: protocol and branch? item->setData (2, Qt::UserRole, tl::to_qstring (d->url)); dependency_changed (item, 2); @@ -254,7 +253,6 @@ SaltGrainPropertiesDialog::update_data () dep.name = tl::to_string (name); dep.version = tl::to_string (version); dep.url = tl::to_string (url); - // @@@ TODO: set protocol and branch m_grain.dependencies ().push_back (dep); } diff --git a/src/tl/tl/tlFileUtils.cc b/src/tl/tl/tlFileUtils.cc index 747d21c6f..752c4085c 100644 --- a/src/tl/tl/tlFileUtils.cc +++ b/src/tl/tl/tlFileUtils.cc @@ -27,6 +27,7 @@ #include "tlEnv.h" #include +#include // Use this define to print debug output // #define FILE_UTILS_VERBOSE @@ -1018,4 +1019,108 @@ get_module_path (void *addr) #endif } +std::string +tmpfile (const std::string &domain) +{ + std::string tmp = tl::get_env ("TMPDIR"); + if (tmp.empty ()) { + tmp = tl::get_env ("TMP"); + } + if (tmp.empty ()) { +#if defined(_WIN32) + throw tl::Exception (tl::to_string (tr ("TMP and TMPDIR not set - cannot create temporary file"))); +#else + tmp = "/tmp"; +#endif + } + + std::string templ = tl::combine_path (tmp, domain + "XXXXXX"); + char *tmpstr = strdup (templ.c_str ()); + +#if defined(_WIN32) + if (_mktemp_s (tmpstr, templ) != 0) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); + } + + // for compatibility with Linux, create the file as an empty one + std::ofstream os (tmpstr); + if (os.bad ()) { + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } + os.close (); +#else + int fd = mkstemp (tmpstr); + if (fd < 0) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } + close (fd); +#endif + + std::string res = tmpstr; + free (tmpstr); + return res; +} + +TemporaryFile::TemporaryFile (const std::string &domain) +{ + m_path = tmpfile (domain); +} + +TemporaryFile::~TemporaryFile () +{ + tl::rm_file (m_path); +} + +std::string +tmpdir (const std::string &domain) +{ + std::string tmp = tl::get_env ("TMPDIR"); + if (tmp.empty ()) { + tmp = tl::get_env ("TMP"); + } + if (tmp.empty ()) { +#if defined(_WIN32) + throw tl::Exception (tl::to_string (tr ("TMP and TMPDIR not set - cannot create temporary file"))); +#else + tmp = "/tmp"; +#endif + } + + std::string templ = tl::combine_path (tmp, domain + "XXXXXX"); + char *tmpstr = strdup (templ.c_str ()); + +#if defined(_WIN32) + if (_mktemp_s (tmpstr, templ) != 0) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); + } + if (! tl::mkdir (tmpstr)) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } +#else + if (mkdtemp (tmpstr) == NULL) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } +#endif + + std::string res = tmpstr; + free (tmpstr); + return res; +} + +TemporaryDirectory::TemporaryDirectory (const std::string &domain) +{ + m_path = tmpdir (domain); +} + +TemporaryDirectory::~TemporaryDirectory () +{ + tl::rm_dir_recursive (m_path); +} + + } diff --git a/src/tl/tl/tlFileUtils.h b/src/tl/tl/tlFileUtils.h index 0649b4078..474ec7e29 100644 --- a/src/tl/tl/tlFileUtils.h +++ b/src/tl/tl/tlFileUtils.h @@ -188,6 +188,68 @@ std::string TL_PUBLIC current_dir (); */ bool TL_PUBLIC chdir (const std::string &path); +/** + * @brief Gets a temporary file path + * + * This function will make a temporary file with a unique name. + * The "domain" string is used as part of the file name as an disambiguator. + * + * The function reads $TMPDIR or $TMP to define the location of the temporary + * directory. On Linux, the default is /tmp. + * + * The file is created and it is the responsibility of the caller to remove + * the file. + */ +std::string TL_PUBLIC tmpfile (const std::string &domain = std::string ()); + +/** + * @brief A class wrapping a temporary file + * + * In the destructor of this class, the temporary file will be deleted again. + */ +class TL_PUBLIC TemporaryFile +{ +public: + TemporaryFile (const std::string &domain = std::string ()); + ~TemporaryFile (); + + const std::string &path () const + { + return m_path; + } + +private: + std::string m_path; +}; + +/** + * @brief Gets a temporary folder path + * + * Similar to "tmpfile", but will create a new, empty folder. Again it is the + * reposibility of the caller to clean up. + */ +std::string TL_PUBLIC tmpdir (const std::string &domain = std::string ()); + +/** + * @brief A class wrapping a temporary directory + * + * In the destructor of this class, the temporary directory will be deleted again. + */ +class TL_PUBLIC TemporaryDirectory +{ +public: + TemporaryDirectory (const std::string &domain = std::string ()); + ~TemporaryDirectory (); + + const std::string &path () const + { + return m_path; + } + +private: + std::string m_path; +}; + /** * @brief This function splits the path into it's components * On Windows, the first component may be the drive prefix ("C:") or diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 95470580f..12298d93f 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -78,12 +78,7 @@ GitObject::GitObject (const std::string &local_path) InitHelper::ensure_initialized (); if (local_path.empty ()) { - // @@@ generic tempnam on Windows/Posix ... - char tmp[] = "/tmp/git2klayoutXXXXXX"; - if (mkdtemp (tmp) == NULL) { - throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder: %s")), (const char *) tmp); - } - m_local_path = tmp; + m_local_path = tl::tmpdir ("git2klayout"); m_is_temp = true; } diff --git a/src/tl/unit_tests/tlFileUtilsTests.cc b/src/tl/unit_tests/tlFileUtilsTests.cc index 4814901c2..768b0943a 100644 --- a/src/tl/unit_tests/tlFileUtilsTests.cc +++ b/src/tl/unit_tests/tlFileUtilsTests.cc @@ -875,3 +875,83 @@ TEST (20) EXPECT_EQ (tl::absolute_file_path ("~"), tl::get_home_path ()); EXPECT_EQ (tl::absolute_file_path (tl::combine_path ("~", "test")), tl::combine_path (tl::get_home_path (), "test")); } + +// tmpfile +TEST (21) +{ + std::string p = tl::tmpfile ("tl_tests"); + EXPECT_EQ (tl::file_exists (p), true); + + std::ofstream os (p); + os << "A test"; + os.close (); + + tl::InputStream is (p); + EXPECT_EQ (is.read_all (), "A test"); + + EXPECT_EQ (tl::rm_file (p), true); + EXPECT_EQ (tl::file_exists (p), false); +} + +// TemporaryFile +TEST (22) +{ + std::string p; + + { + tl::TemporaryFile tf ("tl_tests"); + p = tf.path (); + EXPECT_EQ (tl::file_exists (tf.path ()), true); + + std::ofstream os (tf.path ()); + os << "A test"; + os.close (); + + tl::InputStream is (tf.path ()); + EXPECT_EQ (is.read_all (), "A test"); + } + + EXPECT_EQ (tl::file_exists (p), false); +} + +// tmpdir +TEST (23) +{ + std::string p = tl::tmpdir ("tl_tests"); + EXPECT_EQ (tl::file_exists (p), true); + EXPECT_EQ (tl::is_dir (p), true); + + std::ofstream os (tl::combine_path (p, "test")); + os << "A test"; + os.close (); + + tl::InputStream is (tl::combine_path (p, "test")); + EXPECT_EQ (is.read_all (), "A test"); + + EXPECT_EQ (tl::rm_dir_recursive (p), true); + EXPECT_EQ (tl::file_exists (p), false); +} + +// TemporaryDirectory object +TEST (24) +{ + std::string p; + + { + tl::TemporaryDirectory tmpdir ("tl_tests"); + p = tmpdir.path (); + + EXPECT_EQ (tl::file_exists (p), true); + EXPECT_EQ (tl::is_dir (p), true); + + std::ofstream os (tl::combine_path (p, "test")); + os << "A test"; + os.close (); + + tl::InputStream is (tl::combine_path (p, "test")); + EXPECT_EQ (is.read_all (), "A test"); + } + + EXPECT_EQ (tl::file_exists (p), false); +} + From 0474884a87c36fc4e5dfc441bfe0049c2d947bac Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 1 Nov 2023 18:23:15 +0100 Subject: [PATCH 20/26] Fixed Windows builds --- src/tl/tl/tlFileUtils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tl/tl/tlFileUtils.cc b/src/tl/tl/tlFileUtils.cc index 752c4085c..b25a297f6 100644 --- a/src/tl/tl/tlFileUtils.cc +++ b/src/tl/tl/tlFileUtils.cc @@ -1038,7 +1038,7 @@ tmpfile (const std::string &domain) char *tmpstr = strdup (templ.c_str ()); #if defined(_WIN32) - if (_mktemp_s (tmpstr, templ) != 0) { + if (_mktemp_s (tmpstr, templ.size ()) != 0) { free (tmpstr); throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); } @@ -1092,7 +1092,7 @@ tmpdir (const std::string &domain) char *tmpstr = strdup (templ.c_str ()); #if defined(_WIN32) - if (_mktemp_s (tmpstr, templ) != 0) { + if (_mktemp_s (tmpstr, templ.size ()) != 0) { free (tmpstr); throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); } From 2d2b0f34b34b68636f12e3e1ceca2f2beca37fda Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 1 Nov 2023 19:07:04 +0100 Subject: [PATCH 21/26] Maybe fixing Windows implementation --- src/tl/tl/tlFileUtils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tl/tl/tlFileUtils.cc b/src/tl/tl/tlFileUtils.cc index b25a297f6..28c3741a3 100644 --- a/src/tl/tl/tlFileUtils.cc +++ b/src/tl/tl/tlFileUtils.cc @@ -1038,7 +1038,7 @@ tmpfile (const std::string &domain) char *tmpstr = strdup (templ.c_str ()); #if defined(_WIN32) - if (_mktemp_s (tmpstr, templ.size ()) != 0) { + if (_mktemp_s (tmpstr, templ.size () + 1) != 0) { free (tmpstr); throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); } @@ -1092,7 +1092,7 @@ tmpdir (const std::string &domain) char *tmpstr = strdup (templ.c_str ()); #if defined(_WIN32) - if (_mktemp_s (tmpstr, templ.size ()) != 0) { + if (_mktemp_s (tmpstr, templ.size () + 1) != 0) { free (tmpstr); throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); } From a6a958838d9edd9ab06e90bbcc2f187bc262414a Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 1 Nov 2023 21:16:18 +0100 Subject: [PATCH 22/26] Trying to fix file utils unit test on Windows - cannot remove file when still open? --- src/tl/unit_tests/tlFileUtilsTests.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/tl/unit_tests/tlFileUtilsTests.cc b/src/tl/unit_tests/tlFileUtilsTests.cc index 768b0943a..1397268bd 100644 --- a/src/tl/unit_tests/tlFileUtilsTests.cc +++ b/src/tl/unit_tests/tlFileUtilsTests.cc @@ -886,8 +886,10 @@ TEST (21) os << "A test"; os.close (); - tl::InputStream is (p); - EXPECT_EQ (is.read_all (), "A test"); + { + tl::InputStream is (p); + EXPECT_EQ (is.read_all (), "A test"); + } EXPECT_EQ (tl::rm_file (p), true); EXPECT_EQ (tl::file_exists (p), false); @@ -925,8 +927,10 @@ TEST (23) os << "A test"; os.close (); - tl::InputStream is (tl::combine_path (p, "test")); - EXPECT_EQ (is.read_all (), "A test"); + { + tl::InputStream is (tl::combine_path (p, "test")); + EXPECT_EQ (is.read_all (), "A test"); + } EXPECT_EQ (tl::rm_dir_recursive (p), true); EXPECT_EQ (tl::file_exists (p), false); From 2c37ecdf7c062fd917ba739c97de7b1d60c038ed Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 1 Nov 2023 22:12:32 +0100 Subject: [PATCH 23/26] No libgit2 needed for Python module --- setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 744b46547..1e41400b3 100644 --- a/setup.py +++ b/setup.py @@ -422,15 +422,13 @@ class Config(object): macros = [ ("HAVE_CURL", 1), ("HAVE_EXPAT", 1), + ("HAVE_PNG", 1), + ("HAVE_GIT2", 0), ("KLAYOUT_MAJOR_VERSION", self.major_version()), ("KLAYOUT_MINOR_VERSION", self.minor_version()), ("GSI_ALIAS_INSPECT", 1), ] - if platform.system() == "Darwin" and check_libpng(): - macros += [("HAVE_PNG", 1)] - else: - macros += [("HAVE_PNG", 1)] return macros def minor_version(self): From 9d589b38f5cd9585edb4eebe615237a72f151f2a Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Wed, 1 Nov 2023 23:41:28 +0100 Subject: [PATCH 24/26] Different approach to exclude libgit2 for pymod --- setup.py | 1 - src/tl/tl/tlGit.cc | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1e41400b3..cc195adb5 100644 --- a/setup.py +++ b/setup.py @@ -423,7 +423,6 @@ class Config(object): ("HAVE_CURL", 1), ("HAVE_EXPAT", 1), ("HAVE_PNG", 1), - ("HAVE_GIT2", 0), ("KLAYOUT_MAJOR_VERSION", self.major_version()), ("KLAYOUT_MINOR_VERSION", self.minor_version()), ("GSI_ALIAS_INSPECT", 1), diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 12298d93f..4bbbcdee5 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -20,6 +20,7 @@ */ +#if defined(HAVE_GIT2) #include "tlGit.h" #include "tlFileUtils.h" @@ -396,3 +397,5 @@ GitObject::download_item (const std::string &url, const std::string &file, const } } + +#endif From 08a790d7f707d1ae0001fae3ed8c8d5c87b75e4f Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 5 Nov 2023 22:27:54 +0100 Subject: [PATCH 25/26] Proper handling of credential requests in Git client --- src/tl/tl/tlGit.cc | 9 +++++++++ src/tl/unit_tests/tlGitTests.cc | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 4bbbcdee5..050243abc 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -252,6 +252,14 @@ checkout_branch (git_repository *repo, git_remote *remote, const git_checkout_op check (git_checkout_head (repo, co_opts)); } +int +credentials_cb (git_credential ** /*out*/, const char * /*url*/, const char * /*username*/, unsigned int /*allowed_types*/, void *) +{ + // no credentials aquired + git_error_set_str (GIT_ERROR_NONE, "anonymous access is supported only, but server requests credentials"); + return GIT_EUSER; +} + void GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { @@ -294,6 +302,7 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons fetch_opts.depth = 1; // shallow (single commit) #endif fetch_opts.callbacks.transfer_progress = &fetch_progress; + fetch_opts.callbacks.credentials = &credentials_cb; fetch_opts.callbacks.payload = (void *) &progress; // build refspecs in case they are needed diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc index 82d62dacc..df51035b3 100644 --- a/src/tl/unit_tests/tlGitTests.cc +++ b/src/tl/unit_tests/tlGitTests.cc @@ -27,6 +27,7 @@ #include "tlFileUtils.h" static std::string test_url ("https://github.com/klayout/klayout_git_test.git"); +static std::string test_url_invalid ("https://github.com/klayout/doesnotexist.git"); TEST(1_plain) { @@ -201,5 +202,17 @@ TEST(10_invalid_branch) } } +TEST(11_invalid_url) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + try { + repo.read (test_url_invalid, std::string (), std::string (), std::string ("brxxx")); + EXPECT_EQ (true, false); + } catch (tl::Exception &ex) { + EXPECT_EQ (ex.msg (), "Error cloning Git repo: anonymous access is supported only, but server requests credentials"); + } +} + #endif From 4e00a91e91067a58fe4734dd4573f0130462222b Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 5 Nov 2023 23:04:48 +0100 Subject: [PATCH 26/26] Debugging git package download --- src/lay/lay/laySalt.cc | 14 +++++++------- src/tl/tl/tlGit.cc | 17 ++++++++++++++--- src/tl/tl/tlGit.h | 2 +- src/tl/tl/tlWebDAV.h | 2 +- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/lay/lay/laySalt.cc b/src/lay/lay/laySalt.cc index 81267fa97..efb77844e 100644 --- a/src/lay/lay/laySalt.cc +++ b/src/lay/lay/laySalt.cc @@ -497,25 +497,25 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, double timeout, t } else if (! templ.url ().empty ()) { - if (templ.url ().find ("http:") == 0 || templ.url ().find ("https:") == 0) { + lay::SaltParsedURL purl (templ.url ()); + + if (purl.url ().find ("http:") == 0 || purl.url ().find ("https:") == 0) { // otherwise download from the URL using Git or SVN - lay::SaltParsedURL purl (templ.url ()); - if (purl.protocol () == Git) { #if defined(HAVE_GIT2) - tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); - res = tl::GitObject::download (templ.url (), target.path (), purl.subfolder (), purl.branch (), timeout, callback); + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol (ref='%3', subdir='%4') ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ())).arg (tl::to_qstring (purl.branch ())).arg (tl::to_qstring (purl.subfolder ())); + res = tl::GitObject::download (purl.url (), target.path (), purl.subfolder (), purl.branch (), timeout, callback); #else throw tl::Exception (tl::to_string (QObject::tr ("Unable to install package '%1' - git protocol not compiled in").arg (tl::to_qstring (target.name ())))); #endif } else if (purl.protocol () == WebDAV || purl.protocol () == DefaultProtocol) { - tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); - res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback); + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ())); + res = tl::WebDAVObject::download (purl.url (), target.path (), timeout, callback); } diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 050243abc..6810b07ea 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -384,9 +384,20 @@ GitObject::read (const std::string &org_url, const std::string &org_filter, cons bool GitObject::download (const std::string &url, const std::string &target, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { - GitObject obj (target); - obj.read (url, std::string (), subfolder, branch, timeout, callback); - return false; + try { + + GitObject obj (target); + obj.read (url, std::string (), subfolder, branch, timeout, callback); + + return true; + + } catch (tl::Exception &ex) { + + tl::error << tl::sprintf (tl::to_string (tr ("Error downloading Git repo from %s (subdir '%s', ref '%s')")), url, subfolder, branch); + + return false; + + } } tl::InputStream * diff --git a/src/tl/tl/tlGit.h b/src/tl/tl/tlGit.h index a5d22e342..a99d30449 100644 --- a/src/tl/tl/tlGit.h +++ b/src/tl/tl/tlGit.h @@ -79,7 +79,7 @@ public: * * Sub-directories are created if required. * - * This method throws an exception if the directory structure could + * This method returns false if the directory structure could * not be obtained or downloading of one file failed. * * "branch" is the remote ref to use. This can be a branch name, a tag name, diff --git a/src/tl/tl/tlWebDAV.h b/src/tl/tl/tlWebDAV.h index 5b676a2a7..7a63f6a36 100644 --- a/src/tl/tl/tlWebDAV.h +++ b/src/tl/tl/tlWebDAV.h @@ -145,7 +145,7 @@ public: * * Sub-directories are created if required. * - * This method throws an exception if the directory structure could + * This method returns false if the directory structure could * not be obtained or downloading of one file failed. */ static bool download (const std::string &url, const std::string &target, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);