luke
/
klayout
mirror of https://github.com/KLayout/klayout.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1518 from KLayout/git-for-packages 

Git for packages
This commit is contained in:
Matthias Köfferlein 2023-11-07 21:36:01 +01:00 committed by GitHub
parent 5edbc0b03f 4e00a91e91
commit 041fb53d04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 1982 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build.sh
View File

@ -39,6 +39,7 @@ HAVE_QT=1
HAVE_PNG=0
HAVE_CURL=0
HAVE_EXPAT=0
HAVE_GIT2=1
RUBYINCLUDE=""
RUBYINCLUDE2=""
@ -209,6 +210,9 @@ while [ "$*" != "" ]; do
-libexpat)
HAVE_EXPAT=1
;;
-nolibgit2)
HAVE_GIT2=0
;;
-qt5)
echo "*** WARNING: -qt5 option is ignored - Qt version is auto-detected now."
;;
@ -265,6 +269,7 @@ while [ "$*" != "" ]; do
echo " -libcurl Use libcurl instead of QtNetwork (for Qt<4.7)"
echo " -libexpat Use libexpat instead of QtXml"
echo " -libpng Use libpng instead of Qt for PNG generation"
echo " -nolibgit2 Do not include libgit2 for Git package support"
echo ""
echo "Environment Variables:"
echo ""
@ -495,6 +500,9 @@ fi
if [ $HAVE_PNG != 0 ]; then
echo " Uses libpng for PNG generation"
fi
if [ $HAVE_GIT2 != 0 ]; then
echo " Uses libgit2 for Git access"
fi
if [ "$RPATH" = "" ]; then
RPATH="$BIN"
fi
@ -578,6 +586,7 @@ echo " HAVE_64BIT_COORD=$HAVE_64BIT_COORD"
echo " HAVE_CURL=$HAVE_CURL"
echo " HAVE_PNG=$HAVE_PNG"
echo " HAVE_EXPAT=$HAVE_EXPAT"
echo " HAVE_GIT2=$HAVE_GIT2"
echo " RPATH=$RPATH"
mkdir -p $BUILD
@ -650,6 +659,7 @@ qmake_options=(
HAVE_CURL="$HAVE_CURL"
HAVE_EXPAT="$HAVE_EXPAT"
HAVE_PNG="$HAVE_PNG"
HAVE_GIT2="$HAVE_GIT2"
PREFIX="$BIN"
RPATH="$RPATH"
KLAYOUT_VERSION="$KLAYOUT_VERSION"

8
scripts/makedeb.sh
View File

@ -19,16 +19,16 @@ fi
# TODO: derive this list automatically?
case $target in
ubuntu16)
depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)"
depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libgit2-24 (>= 0.24.0), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)"
;;
ubuntu18)
depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)"
depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libgit2-26 (>= 0.26.0), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)"
;;
ubuntu20)
depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)"
depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libgit2-28 (>= 0.28.4), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)"
;;
ubuntu22)
depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)"
depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libgit2-1.1 (>= 1.1.0), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)"
;;
*)
echo "Unknown target '$target' (given as first argument)"

8
scripts/rpm-data/klayout.spec
View File

@ -53,6 +53,7 @@ Requires: qt5-qttools-devel >= 5.11.1
Requires: ruby >= 2.0.0
Requires: python3 >= 3.6.0
Requires: qt-x11 >= 4.8.5
Requires: libgit2 >= 0.26.8
%define buildopt -j2
%endif
@ -62,7 +63,7 @@ Requires: libcurl >= 7.19.7
Requires: ruby >= 1.8.7
Requires: python >= 2.6.6
Requires: qt-x11 >= 4.6.2
%define buildopt -libcurl -j2
%define buildopt -libcurl -j2 -nolibgit2
%endif
%if "%{target_system}" == "opensuse42_2"
@ -70,7 +71,7 @@ Requires: qt-x11 >= 4.6.2
Requires: ruby2.3 >= 2.3.1
Requires: python3 >= 3.4.6
Requires: libqt4-x11 >= 4.8.6
%define buildopt -j2
%define buildopt -j2 -nolibgit2
%endif
%if "%{target_system}" == "opensuse42_3"
@ -78,13 +79,14 @@ Requires: libqt4-x11 >= 4.8.6
Requires: ruby2.3 >= 2.3.1
Requires: python3 >= 3.4.6
Requires: libqt4-x11 >= 4.8.6
%define buildopt -j2
%define buildopt -j2 -nolibgit2
%endif
%if "%{target_system}" == "opensuse15"
# OpenSuSE Leap 15 requirements
Requires: ruby >= 2.5
Requires: python3 >= 3.6
Requires: libgit2 >= 1.3.0
Requires: libqt5-qtbase >= 5.15.2
Requires: libQt5PrintSupport5 >= 5.15.2
Requires: libQt5Designer5 >= 5.15.2

5
setup.py
View File

@ -422,15 +422,12 @@ class Config(object):
macros = [
("HAVE_CURL", 1),
("HAVE_EXPAT", 1),
("HAVE_PNG", 1),
("KLAYOUT_MAJOR_VERSION", self.major_version()),
("KLAYOUT_MINOR_VERSION", self.minor_version()),
("GSI_ALIAS_INSPECT", 1),
]
if platform.system() == "Darwin" and check_libpng():
macros += [("HAVE_PNG", 1)]
else:
macros += [("HAVE_PNG", 1)]
return macros
def minor_version(self):

2
src/db/db/db.pro
View File

@ -407,7 +407,7 @@ HEADERS = \
dbShapeCollection.h \
dbShapeCollectionUtils.h
!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") {
!equals(HAVE_QT, "0") {
RESOURCES = \
dbResources.qrc \

2
src/drc/drc/drc.pro
View File

@ -13,7 +13,7 @@ HEADERS = \
drcCommon.h \
drcForceLink.h \
!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") {
!equals(HAVE_QT, "0") {
RESOURCES = \
drcResources.qrc
}

9
src/klayout.pri
View File

@ -103,6 +103,15 @@ equals(HAVE_PTHREADS, "1") {
DEFINES += HAVE_PTHREADS
}
equals(HAVE_GIT2, "1") {
!isEmpty(BITS_PATH) {
include($$BITS_PATH/git2/git2.pri)
} else {
LIBS += -lgit2
}
DEFINES += HAVE_GIT2
}
equals(HAVE_RUBY, "1") {
!isEmpty(BITS_PATH) {
include($$BITS_PATH/ruby/ruby.pri)

2
src/lay/lay/lay.pro
View File

@ -33,6 +33,7 @@ HEADERS = \
layResourceHelpProvider.h \
layRuntimeErrorForm.h \
layReaderErrorForm.h \
laySaltParsedURL.h \
laySearchReplaceConfigPage.h \
laySearchReplaceDialog.h \
laySearchReplacePropertiesWidgets.h \
@ -144,6 +145,7 @@ SOURCES = \
layResourceHelpProvider.cc \
layRuntimeErrorForm.cc \
layReaderErrorForm.cc \
laySaltParsedURL.cc \
laySearchReplaceConfigPage.cc \
laySearchReplaceDialog.cc \
laySearchReplacePlugin.cc \

2
src/lay/lay/layApplication.cc
View File

@ -770,7 +770,7 @@ ApplicationBase::init_app ()
size_t local_folders = (lay::get_appdata_path ().empty () ? 0 : 1);
for (std::vector <std::string>::const_iterator p = m_klayout_path.begin (); p != m_klayout_path.end (); ++p) {
if (p - m_klayout_path.begin () < local_folders) {
if (size_t (p - m_klayout_path.begin ()) < local_folders) {
mc->add_path (*p, tl::to_string (QObject::tr ("Local")), std::string (), false);
} else if (m_klayout_path.size () == 1 + local_folders) {
mc->add_path (*p, tl::to_string (QObject::tr ("Global")), std::string (), true);

2
src/lay/lay/layInit.cc
View File

@ -152,7 +152,7 @@ void init (const std::vector<std::string> &_paths)
try {
s_plugins.push_back (do_load_plugin (imp));
modules.insert (*im);
} catch (tl::Exception (&ex)) {
} catch (tl::Exception &ex) {
tl::error << ex.msg ();
}
}

38
src/lay/lay/laySalt.cc
View File

@ -21,11 +21,18 @@
*/
#include "laySalt.h"
#include "laySaltParsedURL.h"
#include "tlString.h"
#include "tlFileUtils.h"
#include "tlLog.h"
#include "tlInternational.h"
#include "tlWebDAV.h"
#include "tlEnv.h"
#if defined(HAVE_GIT2)
# include "tlGit.h"
#endif
#include "lymMacro.h"
#include <QFileInfo>
@ -63,6 +70,13 @@ Salt::root ()
return m_root;
}
bool
Salt::download_package_information () const
{
// $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO
return tl::app_flag ("always-download-package-info") || m_root.sparse ();
}
Salt::flat_iterator
Salt::begin_flat ()
{
@ -483,11 +497,27 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, double timeout, t
} else if (! templ.url ().empty ()) {
if (templ.url ().find ("http:") == 0 || templ.url ().find ("https:") == 0) {
lay::SaltParsedURL purl (templ.url ());
// otherwise download from the URL
tl::info << QObject::tr ("Downloading package from '%1' to '%2' ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ()));
res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback);
if (purl.url ().find ("http:") == 0 || purl.url ().find ("https:") == 0) {
// otherwise download from the URL using Git or SVN
if (purl.protocol () == Git) {
#if defined(HAVE_GIT2)
tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol (ref='%3', subdir='%4') ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ())).arg (tl::to_qstring (purl.branch ())).arg (tl::to_qstring (purl.subfolder ()));
res = tl::GitObject::download (purl.url (), target.path (), purl.subfolder (), purl.branch (), timeout, callback);
#else
throw tl::Exception (tl::to_string (QObject::tr ("Unable to install package '%1' - git protocol not compiled in").arg (tl::to_qstring (target.name ()))));
#endif
} else if (purl.protocol () == WebDAV || purl.protocol () == DefaultProtocol) {
tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ()));
res = tl::WebDAVObject::download (purl.url (), target.path (), timeout, callback);
}
} else {

5
src/lay/lay/laySalt.h
View File

@ -199,6 +199,11 @@ public:
*/
SaltGrains &root ();
/**
* @brief Gets a value indicating whether the collection wants package information to be downloaded always
*/
bool download_package_information () const;
signals:
/**
* @brief A signal triggered before one of the collections changed

16
src/lay/lay/laySaltController.cc
View File

@ -23,6 +23,7 @@
#include "laySaltController.h"
#include "laySaltManagerDialog.h"
#include "laySaltDownloadManager.h"
#include "laySaltParsedURL.h"
#include "layConfig.h"
#include "layMainWindow.h"
#include "layQtTools.h"
@ -142,7 +143,7 @@ SaltController::show_editor ()
{
// while running the dialog, don't watch file events - that would interfere with
// the changes applied by the dialog itself.
lay::BusySection busy_section; // disable file watcher
tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher
mp_salt_dialog->exec ();
}
@ -156,7 +157,7 @@ SaltController::show_editor ()
void
SaltController::sync_file_watcher ()
{
lay::BusySection busy_section; // disable file watcher
tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher
if (m_file_watcher) {
m_file_watcher->clear ();
@ -202,12 +203,19 @@ SaltController::install_packages (const std::vector<std::string> &packages, bool
}
}
if (n.find ("http:") == 0 || n.find ("https:") == 0 || n.find ("file:") == 0 || n[0] == '/' || n[0] == '\\') {
lay::SaltParsedURL purl (n);
const std::string &url = purl.url ();
if (url.find ("http:") == 0 || url.find ("https:") == 0 || url.find ("file:") == 0 || url[0] == '/' || url[0] == '\\') {
// its a URL
manager.register_download (std::string (), std::string (), n, v);
} else {
// its a plain name
manager.register_download (n, std::string (), std::string (), v);
}
}
@ -223,7 +231,7 @@ SaltController::install_packages (const std::vector<std::string> &packages, bool
{
// while running the dialog, don't watch file events - that would interfere with
// the changes applied by the dialog itself.
lay::BusySection busy_section; // disable file watcher
tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher
result = manager.execute (0, m_salt);
}

66
src/lay/lay/laySaltDownloadManager.cc
View File

@ -27,6 +27,7 @@
#include "tlFileUtils.h"
#include "tlWebDAV.h"
#include "tlLog.h"
#include "tlEnv.h"
#include <memory>
#include <QTreeWidgetItem>
@ -310,42 +311,81 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt
++progress;
// Add URL and token from the package index
//
// In order to do so, we try to use the information from that package index as far as possible.
// Downloading a package definition from the original package URL may be expensive in case of
// large GIT repositories.
//
// Downloading is required if:
// - A package download is requested without a name (package can't be looked up in the package index)
// - Or a name is given, but not found in the package index
//
// Downloading can be bypassed if the package index (salt mine) specifies "sparse=false".
// In that case, the package index will have all information about the package.
if (! p->name.empty ()) {
const lay::SaltGrain *g = salt_mine.grain_by_name (p->name);
if (! g) {
if (p->url.empty ()) {
throw tl::Exception (tl::to_string (QObject::tr ("Package '%1' not found in index - cannot resolve download URL").arg (tl::to_qstring (p->name))));
throw tl::Exception (tl::to_string (tr ("Package '%s' not found in index - cannot resolve download URL")), p->name);
}
} else {
if (p->url.empty ()) {
if (tl::verbosity() >= 20) {
tl::log << "Resolved package URL for package " << p->name << ": " << g->url ();
tl::log << tr ("Resolved package URL for package") << " '" << p->name << "': " << g->url ();
}
p->url = g->url ();
}
p->token = g->token ();
p->grain = *g;
p->downloaded = true;
}
}
try {
p->grain = SaltGrain::from_url (p->url);
} catch (tl::Exception &ex) {
throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ()))));
if (! p->downloaded && salt_mine.download_package_information ()) {
// If requested, download package information to complete information from index or dependencies
if (tl::verbosity() >= 10) {
tl::log << tl::sprintf (tl::to_string (tr ("Reading package description for package '%s' from: '%s")), p->name, p->url);
}
try {
p->grain = SaltGrain::from_url (p->url);
p->downloaded = true;
} catch (tl::Exception &ex) {
throw tl::Exception (tl::to_string (tr ("Error fetching spec file for package from '%s': %s")), p->url, ex.msg ());
}
}
if (p->version.empty ()) {
p->version = p->grain.version ();
if (! p->downloaded) {
if (p->name.empty ()) {
throw tl::Exception (tl::to_string (tr ("No name given package from '%s' (from dependencies or command line installation request)")), p->url);
}
if (tl::verbosity() >= 10) {
tl::warn << tl::sprintf (tl::to_string (tr ("Package '%s' not downloaded from: %s. Dependencies may not be resolved.")), p->name, p->url);
}
} else {
if (p->version.empty ()) {
p->version = p->grain.version ();
}
if (p->name.empty ()) {
p->name = p->grain.name ();
}
if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) {
throw tl::Exception (tl::to_string (tr ("Package '%s': package in repository is too old (%s) to satisfy requirements (%s)")), p->name, p->grain.version (), p->version);
}
}
p->name = p->grain.name ();
p->downloaded = true;
if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) {
throw tl::Exception (tl::to_string (QObject::tr ("Package '%1': package in repository is too old (%2) to satisfy requirements (%3)").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (p->grain.version ())).arg (tl::to_qstring (p->version))));
}
}
}

58
src/lay/lay/laySaltGrain.cc
View File

@ -22,11 +22,15 @@
#include "laySaltGrain.h"
#include "laySaltController.h"
#include "laySaltParsedURL.h"
#include "tlString.h"
#include "tlXMLParser.h"
#include "tlHttpStream.h"
#include "tlWebDAV.h"
#include "tlFileUtils.h"
#include "tlWebDAV.h"
#if defined(HAVE_GIT2)
# include "tlGit.h"
#endif
#include <memory>
#include <QDir>
@ -65,7 +69,8 @@ SaltGrain::operator== (const SaltGrain &other) const
m_license == other.m_license &&
m_hidden == other.m_hidden &&
m_authored_time == other.m_authored_time &&
m_installed_time == other.m_installed_time;
m_installed_time == other.m_installed_time
;
}
void
@ -253,18 +258,10 @@ SaltGrain::compare_versions (const std::string &v1, const std::string &v2)
}
}
std::string
SaltGrain::spec_url (const std::string &url)
const std::string &
SaltGrain::spec_file ()
{
std::string res = url;
if (! res.empty()) {
// TODO: use system path separator unless this is a URL
if (res [res.size () - 1] != '/') {
res += "/";
}
res += grain_filename;
}
return res;
return grain_filename;
}
bool
@ -512,14 +509,21 @@ SaltGrain::from_path (const std::string &path)
}
tl::InputStream *
SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStreamCallback *callback)
SaltGrain::stream_from_url (std::string &generic_url, double timeout, tl::InputHttpStreamCallback *callback)
{
if (url.empty ()) {
if (generic_url.empty ()) {
throw tl::Exception (tl::to_string (QObject::tr ("No download link available")));
}
if (tl::verbosity () >= 20) {
tl::info << tr ("Downloading package info from ") << generic_url;
}
lay::SaltParsedURL purl (generic_url);
const std::string &url = purl.url ();
// base relative URL's on the salt mine URL
if (url.find ("http:") != 0 && url.find ("https:") != 0 && url.find ("file:") != 0 && !url.empty() && url[0] != '/' && url[0] != '\\' && lay::SaltController::instance ()) {
if (purl.protocol () == lay::DefaultProtocol && url.find ("http:") != 0 && url.find ("https:") != 0 && url.find ("file:") != 0 && !url.empty() && url[0] != '/' && url[0] != '\\' && lay::SaltController::instance ()) {
// replace the last component ("repository.xml") by the given path
QUrl sami_url (tl::to_qstring (lay::SaltController::instance ()->salt_mine_url ()));
@ -529,15 +533,27 @@ SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStrea
}
sami_url.setPath (path_comp.join (QString::fromUtf8 ("/")));
url = tl::to_string (sami_url.toString ());
// return the full path as a file path, not an URL
generic_url = tl::to_string (sami_url.toString ());
}
std::string spec_url = SaltGrain::spec_url (url);
if (spec_url.find ("http:") == 0 || spec_url.find ("https:") == 0) {
return tl::WebDAVObject::download_item (spec_url, timeout, callback);
if (url.find ("http:") == 0 || url.find ("https:") == 0) {
if (purl.protocol () == lay::Git) {
#if defined(HAVE_GIT2)
return tl::GitObject::download_item (url, SaltGrain::spec_file (), purl.subfolder (), purl.branch (), timeout, callback);
#else
throw tl::Exception (tl::to_string (QObject::tr ("Cannot download from Git - Git support not compiled in")));
#endif
} else {
return tl::WebDAVObject::download_item (url + "/" + SaltGrain::spec_file (), timeout, callback);
}
} else {
return new tl::InputStream (spec_url);
return new tl::InputStream (url);
}
}

11
src/lay/lay/laySaltGrain.h
View File

@ -49,6 +49,9 @@ namespace lay
*/
struct SaltGrainDependency
{
SaltGrainDependency ()
{ }
std::string name;
std::string url;
std::string version;
@ -466,6 +469,8 @@ public:
* This method will return a grain constructed from the downloaded data.
* The data is read from "URL/grain.xml". This method will throw an
* exception if an error occurs during reading.
*
* CAUTION: with GIT protocol and large repositories, this function may be very expensive.
*/
static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);
@ -474,13 +479,15 @@ public:
* The stream is a new'd object and needs to be deleted by the caller.
* "url" is the download URL on input and gets modified to match the
* actual URL if it is a relative one.
*
* CAUTION: with GIT protocol and large repositories, this function may be very expensive.
*/
static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);
/**
* @brief Forms the spec file download URL from a given download URL
* @brief Gets the name of the spec file ("grain.xml")
*/
static std::string spec_url (const std::string &url);
static const std::string &spec_file ();
/**
* @brief Returns a value indicating whether the given path represents is a grain

11
src/lay/lay/laySaltGrainPropertiesDialog.cc
View File

@ -25,6 +25,7 @@
#include "tlString.h"
#include "tlExceptions.h"
#include "tlHttpStream.h"
#include "tlEnv.h"
#include <QFileDialog>
#include <QFileInfo>
@ -43,6 +44,14 @@ namespace lay
// ----------------------------------------------------------------------------------------------------
static bool download_package_information ()
{
// $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO
return tl::app_flag ("always-download-package-info");
}
// ----------------------------------------------------------------------------------------------------
/**
* @brief A delegate for editing a field of the dependency list
*/
@ -583,7 +592,7 @@ SaltGrainPropertiesDialog::accept ()
<< tr ("If the dependency package has a version itself, the version is automatically set to its current version.");
}
if (!d->url.empty ()) {
if (!d->url.empty () && download_package_information ()) {
SaltGrain gdep;
try {
gdep = SaltGrain::from_url (d->url);

11
src/lay/lay/laySaltGrains.cc
View File

@ -34,6 +34,7 @@ namespace lay
{
SaltGrains::SaltGrains ()
: m_sparse (true)
{
// .. nothing yet ..
}
@ -54,6 +55,12 @@ SaltGrains::set_name (const std::string &n)
m_name = n;
}
void
SaltGrains::set_sparse (const bool &f)
{
m_sparse = f;
}
void
SaltGrains::set_title (const std::string &t)
{
@ -302,6 +309,7 @@ SaltGrains::consolidate ()
static tl::XMLElementList s_group_struct =
tl::make_member (&SaltGrains::name, &SaltGrains::set_name, "name") +
tl::make_member (&SaltGrains::sparse, &SaltGrains::set_sparse, "sparse") +
tl::make_member (&SaltGrains::include, "include") +
tl::make_element (&SaltGrains::begin_collections, &SaltGrains::end_collections, &SaltGrains::add_collection, "group", &s_group_struct) +
tl::make_element (&SaltGrains::begin_grains, &SaltGrains::end_grains, &SaltGrains::add_grain, "salt-grain", SaltGrain::xml_elements ());
@ -354,6 +362,9 @@ SaltGrains::include (const std::string &src_in)
lay::SaltGrains g;
g.load (src);
if (g.sparse ()) {
m_sparse = true;
}
m_collections.splice (m_collections.end (), g.m_collections);
m_grains.splice (m_grains.end (), g.m_grains);

19
src/lay/lay/laySaltGrains.h
View File

@ -78,6 +78,24 @@ public:
*/
void set_name (const std::string &p);
/**
* @brief Gets a value indicating that the information in the grain collection is sparse
*
* If this flag is set to true (the default), the information in the collection needs
* to be completed by pulling the original definition of the grain for the grain's URL.
* If the flag is false, the information is complete and reflects the grain's original
* definition.
*/
const bool &sparse () const
{
return m_sparse;
}
/**
* @brief Sets a value indicating that the information in the grain collection is sparse
*/
void set_sparse (const bool &f);
/**
* @brief Gets the title of the grain collection
*
@ -225,6 +243,7 @@ private:
collections_type m_collections;
grains_type m_grains;
std::string m_url;
bool m_sparse;
};
}

204
src/lay/lay/laySaltManagerDialog.cc
View File

@ -30,11 +30,13 @@
#include "ui_SaltGrainTemplateSelectionDialog.h"
#include "tlString.h"
#include "tlExceptions.h"
#include "tlEnv.h"
#include "rba.h"
#include "pya.h"
#include <QTextDocument>
#include <QApplication>
#include <QPainter>
#include <QDir>
#include <QTextStream>
@ -289,7 +291,9 @@ SaltManagerDialog::SaltManagerDialog (QWidget *parent, lay::Salt *salt, const st
: QDialog (parent),
m_salt_mine_url (salt_mine_url),
dm_update_models (this, &SaltManagerDialog::update_models), m_current_tab (-1),
mp_downloaded_target (0)
mp_downloaded_target (0),
dm_mine_update_selected_changed (this, &SaltManagerDialog::do_mine_update_selected_changed),
dm_mine_new_selected_changed (this, &SaltManagerDialog::do_mine_new_selected_changed)
{
Ui::SaltManagerDialog::setupUi (this);
mp_properties_dialog = new lay::SaltGrainPropertiesDialog (this);
@ -845,6 +849,8 @@ SaltManagerDialog::salt_mine_about_to_change ()
void
SaltManagerDialog::refresh ()
{
m_salt_grain_cache.clear ();
if (! m_salt_mine_url.empty ()) {
tl::log << tl::to_string (tr ("Downloading package repository from %1").arg (tl::to_qstring (m_salt_mine_url)));
@ -1094,8 +1100,12 @@ SaltManagerDialog::current_grains ()
void
SaltManagerDialog::mine_update_selected_changed ()
{
BEGIN_PROTECTED
dm_mine_update_selected_changed ();
}
void
SaltManagerDialog::do_mine_update_selected_changed ()
{
SaltModel *model = dynamic_cast <SaltModel *> (salt_mine_view_update->model ());
tl_assert (model != 0);
@ -1108,15 +1118,17 @@ BEGIN_PROTECTED
details_update_frame->setEnabled (g != 0);
get_remote_grain_info (g, details_update_text);
END_PROTECTED
}
void
SaltManagerDialog::mine_new_selected_changed ()
{
BEGIN_PROTECTED
dm_mine_new_selected_changed ();
}
void
SaltManagerDialog::do_mine_new_selected_changed ()
{
SaltModel *model = dynamic_cast <SaltModel *> (salt_mine_view_new->model ());
tl_assert (model != 0);
@ -1129,8 +1141,73 @@ BEGIN_PROTECTED
details_new_frame->setEnabled (g != 0);
get_remote_grain_info (g, details_new_text);
}
namespace
{
/**
* @brief A callback to keep the UI alive (mainly used for Git grain retrieval)
*/
class ProcessEventCallback
: public tl::InputHttpStreamCallback
{
public:
virtual void wait_for_input ()
{
QApplication::processEvents (QEventLoop::ExcludeUserInputEvents);
}
};
class FetchGrainInfoProgressAdaptor
: public tl::ProgressAdaptor
{
public:
FetchGrainInfoProgressAdaptor (SaltGrainDetailsTextWidget *details, const std::string &name, const QString &html)
: mp_details (details), m_name (name), m_html (html)
{
mp_details->setHtml (m_html.arg (QString ()));
m_counter = 0;
}
virtual void yield (tl::Progress *progress)
{
QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents, 100);
++m_counter;
std::string all_dots = "..........";
m_counter = m_counter % all_dots.size ();
std::string dots = std::string (all_dots, 0, m_counter);
mp_details->setHtml (m_html.arg (tl::to_qstring (tl::sprintf (tl::to_string (tr ("Downloading %.0f%% %s")), progress->value (), dots))));
}
virtual void trigger (tl::Progress * /*progress*/)
{
// .. nothing yet ..
}
void error ()
{
mp_details->setHtml (m_html.arg (QString ()));
}
void success ()
{
mp_details->setHtml (m_html.arg (QString ()));
}
bool is_aborted () const
{
return false;
}
private:
lay::SaltGrainDetailsTextWidget *mp_details;
std::string m_name;
QString m_html;
unsigned int m_counter;
};
END_PROTECTED
}
void
@ -1141,68 +1218,104 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex
return;
}
tl_assert (m_downloaded_grain.get () == 0);
m_downloaded_grain.reset (0);
if (m_downloaded_grain_reader.get ()) {
// NOTE: don't delete the reader in the slot it triggered
m_downloaded_grain_reader->close ();
}
m_downloaded_grain_reader.reset (0);
mp_downloaded_target = details;
m_salt_mine_grain.reset (new lay::SaltGrain (*g));
// Download actual grain definition file
try {
if (m_salt_mine.download_package_information () && m_salt_mine.grain_by_name (g->name ())) {
if (g->url ().empty ()) {
throw tl::Exception (tl::to_string (tr ("No download link available")));
// Download actual grain definition file
try {
if (g->url ().empty ()) {
throw tl::Exception (tl::to_string (tr ("No download link available")));
}
QString html = tr (
"<html>"
"<body>"
"<font color=\"#c0c0c0\">"
"<h2>Fetching Package Definition ...</h2>"
"<p><b>URL</b>: %1</p>"
"<p>%2</p>"
"</font>"
"</body>"
"</html>"
)
.arg (tl::to_qstring (g->url ()));
details->setHtml (html.arg (QString ()));
FetchGrainInfoProgressAdaptor pa (details, g->name (), html);
std::string url = g->url ();
auto sg = m_salt_grain_cache.find (url);
if (sg == m_salt_grain_cache.end ()) {
m_downloaded_grain.reset (new SaltGrain ());
m_downloaded_grain->set_url (url);
// NOTE: stream_from_url may modify the URL, hence we set it again
ProcessEventCallback callback;
m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback));
m_downloaded_grain->set_url (url);
tl::InputHttpStream *http = dynamic_cast<tl::InputHttpStream *> (m_downloaded_grain_reader->base ());
if (http) {
// async reading on HTTP
http->ready ().add (this, &SaltManagerDialog::data_ready);
http->send ();
} else {
data_ready ();
}
} else {
m_downloaded_grain.reset (new SaltGrain (sg->second));
data_ready ();
}
} catch (tl::Exception &ex) {
show_error (ex);
}
QString html = tr (
"<html>"
"<body>"
"<font color=\"#c0c0c0\">"
"<h2>Fetching Package Definition ...</h2>"
"<p><b>URL</b>: %1</p>"
"</font>"
"</body>"
"</html>"
)
.arg (tl::to_qstring (SaltGrain::spec_url (g->url ())));
} else {
details->setHtml (html);
// Download denied - take information from index
m_downloaded_grain.reset (new SaltGrain (*g));
data_ready ();
std::string url = g->url ();
m_downloaded_grain.reset (new SaltGrain ());
m_downloaded_grain->set_url (url);
// NOTE: stream_from_url may modify the URL, hence we set it again
m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url));
m_downloaded_grain->set_url (url);
tl::InputHttpStream *http = dynamic_cast<tl::InputHttpStream *> (m_downloaded_grain_reader->base ());
if (http) {
// async reading on HTTP
http->ready ().add (this, &SaltManagerDialog::data_ready);
http->send ();
} else {
data_ready ();
}
} catch (tl::Exception &ex) {
show_error (ex);
}
}
void
SaltManagerDialog::data_ready ()
{
if (! m_salt_mine_grain.get () || ! m_downloaded_grain.get () || ! m_downloaded_grain_reader.get () || ! mp_downloaded_target) {
if (! m_salt_mine_grain.get () || ! m_downloaded_grain.get () || ! mp_downloaded_target) {
return;
}
// Load the grain file (save URL as it is overwritten by the grain.xml content)
std::string url = m_downloaded_grain->url ();
m_downloaded_grain->load (*m_downloaded_grain_reader);
m_downloaded_grain->set_url (url);
if (m_downloaded_grain_reader.get ()) {
m_downloaded_grain->load (*m_downloaded_grain_reader);
m_downloaded_grain->set_url (url);
}
// commit to cache
if (m_salt_grain_cache.find (url) == m_salt_grain_cache.end ()) {
m_salt_grain_cache [url] = *m_downloaded_grain;
}
try {
@ -1223,6 +1336,7 @@ SaltManagerDialog::data_ready ()
m_salt_mine_grain.reset (0);
} catch (tl::Exception &ex) {
m_downloaded_grain.reset (0);
show_error (ex);
}
}

5
src/lay/lay/laySaltManagerDialog.h
View File

@ -188,6 +188,9 @@ private:
std::unique_ptr<lay::SaltGrain> m_downloaded_grain, m_salt_mine_grain;
SaltGrainDetailsTextWidget *mp_downloaded_target;
std::unique_ptr<tl::InputStream> m_salt_mine_reader;
tl::DeferredMethod<SaltManagerDialog> dm_mine_update_selected_changed;
tl::DeferredMethod<SaltManagerDialog> dm_mine_new_selected_changed;
std::map<std::string, lay::SaltGrain> m_salt_grain_cache;
SaltGrain *current_grain ();
std::vector<lay::SaltGrain *> current_grains ();
@ -199,6 +202,8 @@ private:
void show_error (tl::Exception &ex);
void salt_mine_download_started ();
void salt_mine_download_finished ();
void do_mine_update_selected_changed ();
void do_mine_new_selected_changed ();
};
}

150
src/lay/lay/laySaltParsedURL.cc Normal file
View File

@ -0,0 +1,150 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2023 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "laySaltParsedURL.h"
#include "laySaltGrain.h"
#include "tlString.h"
namespace lay
{
static void
parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::string &subfolder)
{
const char *org_str = ex.get ();
std::string w;
// protocol (http:)
ex.try_read_word (w, "") && ex.test (":");
while (! ex.at_end () && ex.test ("/")) {
;
}
// server ("www.klayout.de")
while (! ex.at_end () && (*ex != '/' && *ex != '+' && *ex != '[')) {
++ex;
}
while (! ex.at_end ()) {
++ex;
// next component
const char *c1 = ex.get ();
while (! ex.at_end () && (*ex != '/' && *ex != '+' && *ex != '[')) {
++ex;
}
const char *c2 = ex.get ();
std::string comp (c1, c2 - c1);
if ((! ex.at_end () && (*ex == '+' || *ex == '[')) || comp.find (".git") == comp.size () - 4) {
// subfolder starts here
break;
}
}
url = std::string (org_str, ex.get () - org_str);
if (ex.at_end ()) {
return;
}
// skip URL/subfolder separator
if (*ex == '/') {
while (! ex.at_end () && *ex == '/') {
++ex;
}
} else if (*ex == '+') {
++ex;
}
// subfolders
{
const char *c1 = ex.get ();
while (! ex.at_end () && *ex != '[') {
++ex;
}
const char *c2 = ex.get ();
subfolder = std::string (c1, c2 - c1);
}
if (! ex.at_end () && *ex == '[') {
// explicit branch
++ex;
const char *c1 = ex.get ();
while (! ex.at_end () && *ex != ']') {
++ex;
}
const char *c2 = ex.get ();
branch = std::string (c1, c2 - c1);
} else if (! subfolder.empty ()) {
// SVN emulation
auto parts = tl::split (subfolder, "/");
if (parts.size () >= 1 && parts.front () == "trunk") {
branch = "HEAD";
parts.erase (parts.begin ());
subfolder = tl::join (parts, "/");
} else if (parts.size () >= 2 && parts.front () == "tags") {
branch = "refs/tags/" + parts[1];
parts.erase (parts.begin (), parts.begin () + 2);
subfolder = tl::join (parts, "/");
} else if (parts.size () >= 2 && parts.front () == "branches") {
branch = "refs/heads/" + parts[1];
parts.erase (parts.begin (), parts.begin () + 2);
subfolder = tl::join (parts, "/");
}
}
}
SaltParsedURL::SaltParsedURL (const std::string &url)
: m_protocol (lay::DefaultProtocol)
{
tl::Extractor ex (url.c_str ());
if (ex.test ("svn") && ex.test ("+")) {
m_protocol = lay::WebDAV;
m_url = ex.get ();
return;
}
ex = tl::Extractor (url.c_str ());
if (ex.test ("git") && ex.test ("+")) {
m_protocol = lay::Git;
parse_git_url (ex, m_url, m_branch, m_subfolder);
return;
}
m_url = url;
}
}

115
src/lay/lay/laySaltParsedURL.h Normal file
View File

@ -0,0 +1,115 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2023 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef HDR_laySaltParsedURL
#define HDR_laySaltParsedURL
#include "layCommon.h"
#include "laySaltGrain.h"
namespace lay
{
/**
* @brief An enum describing the protocol to use for download
*/
enum Protocol {
DefaultProtocol = 0,
WebDAV = 1,
Git = 2
};
/**
* @brief A class representing a SaltGrain URL
*
* The URL is parsed into protocol, branch, URL and subfolder if applicable.
* Some heuristics is applied to decompose parts.
*
* SVN URLs:
* https://server.com/repo/trunk -> protocol=DefaultProtocol, url="https://server.com/repo/trunk", branch="", subfolder=""
* svn+https://server.com/repo/trunk -> protocol=WebDAV, url="https://server.com/repo/trunk", branch="", subfolder=""
*
* Git URL heuristics:
* git+https://server.com/repo.git -> protocol=Git, url="https://server.com/repo.git", branch="", subfolder=""
* git+https://server.com/repo.git/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="", subfolder="sub/folder"
* git+https://server.com/repo+sub/folder -> protocol=Git, url="https://server.com/repo", branch="", subfolder="sub/folder"
* git+https://server.com/repo.git[v1.0] -> protocol=Git, url="https://server.com/repo.git", branch="v1.0", subfolder=""
* git+https://server.com/repo.git/sub/folder[refs/tags/1.0] -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.0", subfolder="sub/folder"
* git+https://server.com/repo.git/trunk -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder=""
* git+https://server.com/repo.git/trunk/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="sub/folder"
* git+https://server.com/repo.git/branches/release -> protocol=Git, url="https://server.com/repo.git", branch="refs/heads/release", subfolder=""
* git+https://server.com/repo.git/tags/1.9 -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder=""
* git+https://server.com/repo.git/tags/1.9/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="sub/folder"
*/
class LAY_PUBLIC SaltParsedURL
{
public:
/**
* @brief Constructor: creates an URL from the given generic URL string
*
* This will decompose the URL into the parts and fill protocol, branch and subfolder fields.
*/
SaltParsedURL (const std::string &url);
/**
* @brief Gets the basic URL
*/
const std::string &url () const
{
return m_url;
}
/**
* @brief Gets the subfolder string
*/
const std::string &subfolder () const
{
return m_subfolder;
}
/**
* @brief Gets the branch string
*/
const std::string &branch () const
{
return m_branch;
}
/**
* @brief Gets the protocol
*/
lay::Protocol protocol () const
{
return m_protocol;
}
private:
std::string m_url;
std::string m_branch;
std::string m_subfolder;
lay::Protocol m_protocol;
};
}
#endif

141
src/lay/unit_tests/laySaltParsedURLTests.cc Normal file
View File

@ -0,0 +1,141 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2023 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "laySaltParsedURL.h"
#include "tlUnitTest.h"
TEST (1_Basic)
{
lay::SaltParsedURL purl ("https://server.com/repo/trunk");
EXPECT_EQ (purl.protocol () == lay::DefaultProtocol, true);
EXPECT_EQ (purl.url (), "https://server.com/repo/trunk");
EXPECT_EQ (purl.branch (), "");
EXPECT_EQ (purl.subfolder (), "");
}
TEST (2_SVN)
{
lay::SaltParsedURL purl ("svn+https://server.com/repo/trunk");
EXPECT_EQ (purl.protocol () == lay::WebDAV, true);
EXPECT_EQ (purl.url (), "https://server.com/repo/trunk");
EXPECT_EQ (purl.branch (), "");
EXPECT_EQ (purl.subfolder (), "");
}
TEST (10_GitBasic)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "");
EXPECT_EQ (purl.subfolder (), "");
}
TEST (11_GitSubFolder)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "");
EXPECT_EQ (purl.subfolder (), "sub/folder");
}
TEST (12_GitExplicitBranch)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git[v1.0]");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "v1.0");
EXPECT_EQ (purl.subfolder (), "");
}
TEST (13_GitExplicitBranchAndSubFolder)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder[refs/tags/1.0]");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "refs/tags/1.0");
EXPECT_EQ (purl.subfolder (), "sub/folder");
}
TEST (14_GitExplicitBranchAndExplicitSubFolder)
{
lay::SaltParsedURL purl ("git+https://server.com/repo+sub/folder[refs/tags/1.0]");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo");
EXPECT_EQ (purl.branch (), "refs/tags/1.0");
EXPECT_EQ (purl.subfolder (), "sub/folder");
}
TEST (15_GitSVNEmulationTrunk)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git/trunk");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "HEAD");
EXPECT_EQ (purl.subfolder (), "");
}
TEST (16_GitSVNEmulationTrunkWithSubFolder)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git/trunk/sub/folder");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "HEAD");
EXPECT_EQ (purl.subfolder (), "sub/folder");
}
TEST (17_GitSVNEmulationBranch)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git/branches/xyz");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "refs/heads/xyz");
EXPECT_EQ (purl.subfolder (), "");
}
TEST (18_GitSVNEmulationTag)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git/tags/1.9");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "refs/tags/1.9");
EXPECT_EQ (purl.subfolder (), "");
}
TEST (19_GitSVNEmulationTagWithSubFolder)
{
lay::SaltParsedURL purl ("git+https://server.com/repo.git/tags/1.9/sub/folder");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://server.com/repo.git");
EXPECT_EQ (purl.branch (), "refs/tags/1.9");
EXPECT_EQ (purl.subfolder (), "sub/folder");
}
TEST (20_Example1)
{
lay::SaltParsedURL purl ("git+https://github.com/my-user/test-core[refs/tags/v1.1.0]");
EXPECT_EQ (purl.protocol () == lay::Git, true);
EXPECT_EQ (purl.url (), "https://github.com/my-user/test-core");
EXPECT_EQ (purl.branch (), "refs/tags/v1.1.0");
EXPECT_EQ (purl.subfolder (), "");
}

1
src/lay/unit_tests/unit_tests.pro
View File

@ -9,6 +9,7 @@ include($$PWD/../../lib_ut.pri)
SOURCES = \
laySalt.cc \
layHelpIndexTest.cc \
laySaltParsedURLTests.cc \
laySessionTests.cc
INCLUDEPATH += $$LAY_INC $$TL_INC $$LAYBASIC_INC $$LAYUI_INC $$LAYVIEW_INC $$DB_INC $$GSI_INC $$ANT_INC $$IMG_INC $$RDB_INC

82
src/lib/lib/lib.pro
View File

@ -1,40 +1,42 @@
DESTDIR = $$OUT_PWD/../..
TARGET = klayout_lib
include($$PWD/../../lib.pri)
DEFINES += MAKE_LIB_LIBRARY
HEADERS = \
libBasicArc.h \
libBasicCircle.h \
libBasicDonut.h \
libBasicEllipse.h \
libBasicPie.h \
libBasicRoundPath.h \
libBasicRoundPolygon.h \
libBasicStrokedPolygon.h \
libBasicText.h \
libForceLink.h
SOURCES = \
libForceLink.cc \
libBasic.cc \
libBasicArc.cc \
libBasicCircle.cc \
libBasicDonut.cc \
libBasicEllipse.cc \
libBasicPie.cc \
libBasicRoundPath.cc \
libBasicRoundPolygon.cc \
libBasicStrokedPolygon.cc \
libBasicText.cc
RESOURCES = \
libResources.qrc
INCLUDEPATH += $$TL_INC $$GSI_INC $$DB_INC
DEPENDPATH += $$TL_INC $$GSI_INC $$DB_INC
LIBS += -L$$DESTDIR -lklayout_gsi -lklayout_tl -lklayout_db
DESTDIR = $$OUT_PWD/../..
TARGET = klayout_lib
include($$PWD/../../lib.pri)
DEFINES += MAKE_LIB_LIBRARY
HEADERS = \
libBasicArc.h \
libBasicCircle.h \
libBasicDonut.h \
libBasicEllipse.h \
libBasicPie.h \
libBasicRoundPath.h \
libBasicRoundPolygon.h \
libBasicStrokedPolygon.h \
libBasicText.h \
libForceLink.h
SOURCES = \
libForceLink.cc \
libBasic.cc \
libBasicArc.cc \
libBasicCircle.cc \
libBasicDonut.cc \
libBasicEllipse.cc \
libBasicPie.cc \
libBasicRoundPath.cc \
libBasicRoundPolygon.cc \
libBasicStrokedPolygon.cc \
libBasicText.cc
!equals(HAVE_QT, "0") {
RESOURCES = \
libResources.qrc
}
INCLUDEPATH += $$TL_INC $$GSI_INC $$DB_INC
DEPENDPATH += $$TL_INC $$GSI_INC $$DB_INC
LIBS += -L$$DESTDIR -lklayout_gsi -lklayout_tl -lklayout_db

2
src/lvs/lvs/lvs.pro
View File

@ -13,7 +13,7 @@ HEADERS = \
lvsCommon.h \
lvsForceLink.h \
!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") {
!equals(HAVE_QT, "0") {
RESOURCES = \
lvsResources.qrc
}

2
src/rba/unit_tests/unit_tests.pro
View File

@ -14,7 +14,7 @@ DEPENDPATH += $$RBA_INC $$TL_INC $$DB_INC $$GSI_INC
LIBS += -L$$DESTDIR_UT -lklayout_rba -lklayout_tl -lklayout_db -lklayout_gsi
!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") {
!equals(HAVE_QT, "0") {
RESOURCES = \
rba_unit_tests.qrc
}

12
src/tl/tl/tl.pro
View File

@ -121,7 +121,17 @@ HEADERS = \
tlUniqueName.h \
tlRecipe.h \
tlSelect.h \
tlEnv.h
tlEnv.h
equals(HAVE_GIT2, "1") {
HEADERS += \
tlGit.h
SOURCES += \
tlGit.cc
}
equals(HAVE_CURL, "1") {

17
src/tl/tl/tlFileSystemWatcher.h
View File

@ -140,6 +140,23 @@ private:
std::map<std::string, FileEntry>::iterator m_iter;
};
/**
* @brief A class employing RIIA for locking the file system watcher
*/
class TL_PUBLIC FileSystemWatcherDisabled
{
public:
FileSystemWatcherDisabled ()
{
tl::FileSystemWatcher::global_enable (false);
}
~FileSystemWatcherDisabled ()
{
tl::FileSystemWatcher::global_enable (true);
}
};
}
#endif

147
src/tl/tl/tlFileUtils.cc
View File

@ -27,6 +27,7 @@
#include "tlEnv.h"
#include <cctype>
#include <fstream>
// Use this define to print debug output
// #define FILE_UTILS_VERBOSE
@ -584,6 +585,48 @@ cp_dir_recursive (const std::string &source, const std::string &target)
return true;
}
bool
mv_dir_recursive (const std::string &source, const std::string &target)
{
std::vector<std::string> entries;
std::string path = tl::absolute_file_path (source);
std::string path_to = tl::absolute_file_path (target);
bool error = false;
entries = dir_entries (path, false /*without_files*/, true /*with_dirs*/);
for (std::vector<std::string>::const_iterator e = entries.begin (); e != entries.end (); ++e) {
std::string tc = tl::combine_path (path_to, *e);
if (! mkpath (tc)) {
#if defined(FILE_UTILS_VERBOSE)
tl::error << tr ("Unable to create target directory: ") << tc;
#endif
error = true;
} else if (! mv_dir_recursive (tl::combine_path (path, *e), tc)) {
error = true;
}
}
entries = dir_entries (path, true /*with_files*/, false /*without_dirs*/);
for (std::vector<std::string>::const_iterator e = entries.begin (); e != entries.end (); ++e) {
if (! tl::rename_file (tl::combine_path (path, *e), tl::combine_path (path_to, *e))) {
#if defined(FILE_UTILS_VERBOSE)
tl::error << tr ("Unable to move file from ") << tl::combine_path (path, *e) << tr (" to ") << tl::combine_path (path_to, *e);
#endif
error = true;
}
}
if (! tl::rm_dir (path)) {
#if defined(FILE_UTILS_VERBOSE)
tl::error << tr ("Unable to remove folder ") << path;
#endif
error = true;
}
return ! error;
}
std::string absolute_path (const std::string &s)
{
std::vector<std::string> parts = split_path (absolute_file_path (s));
@ -976,4 +1019,108 @@ get_module_path (void *addr)
#endif
}
std::string
tmpfile (const std::string &domain)
{
std::string tmp = tl::get_env ("TMPDIR");
if (tmp.empty ()) {
tmp = tl::get_env ("TMP");
}
if (tmp.empty ()) {
#if defined(_WIN32)
throw tl::Exception (tl::to_string (tr ("TMP and TMPDIR not set - cannot create temporary file")));
#else
tmp = "/tmp";
#endif
}
std::string templ = tl::combine_path (tmp, domain + "XXXXXX");
char *tmpstr = strdup (templ.c_str ());
#if defined(_WIN32)
if (_mktemp_s (tmpstr, templ.size () + 1) != 0) {
free (tmpstr);
throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp);
}
// for compatibility with Linux, create the file as an empty one
std::ofstream os (tmpstr);
if (os.bad ()) {
throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp);
}
os.close ();
#else
int fd = mkstemp (tmpstr);
if (fd < 0) {
free (tmpstr);
throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp);
}
close (fd);
#endif
std::string res = tmpstr;
free (tmpstr);
return res;
}
TemporaryFile::TemporaryFile (const std::string &domain)
{
m_path = tmpfile (domain);
}
TemporaryFile::~TemporaryFile ()
{
tl::rm_file (m_path);
}
std::string
tmpdir (const std::string &domain)
{
std::string tmp = tl::get_env ("TMPDIR");
if (tmp.empty ()) {
tmp = tl::get_env ("TMP");
}
if (tmp.empty ()) {
#if defined(_WIN32)
throw tl::Exception (tl::to_string (tr ("TMP and TMPDIR not set - cannot create temporary file")));
#else
tmp = "/tmp";
#endif
}
std::string templ = tl::combine_path (tmp, domain + "XXXXXX");
char *tmpstr = strdup (templ.c_str ());
#if defined(_WIN32)
if (_mktemp_s (tmpstr, templ.size () + 1) != 0) {
free (tmpstr);
throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp);
}
if (! tl::mkdir (tmpstr)) {
free (tmpstr);
throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp);
}
#else
if (mkdtemp (tmpstr) == NULL) {
free (tmpstr);
throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp);
}
#endif
std::string res = tmpstr;
free (tmpstr);
return res;
}
TemporaryDirectory::TemporaryDirectory (const std::string &domain)
{
m_path = tmpdir (domain);
}
TemporaryDirectory::~TemporaryDirectory ()
{
tl::rm_dir_recursive (m_path);
}
}

71
src/tl/tl/tlFileUtils.h
View File

@ -52,11 +52,18 @@ bool TL_PUBLIC rm_dir_recursive (const std::string &path);
bool TL_PUBLIC mkpath (const std::string &path);
/**
* @brief Recursively remove the given directory, the files from that directory and all sub-directories (version with std::string)
* @brief Recursively copy the given directory
* @return True, if successful. false otherwise.
*/
bool TL_PUBLIC cp_dir_recursive (const std::string &source, const std::string &target);
/**
* @brief Recursively move the contents of the given directory
* @return True, if successful. false otherwise.
* After this operation, the source directory is deleted.
*/
bool TL_PUBLIC mv_dir_recursive (const std::string &source, const std::string &target);
/**
* @brief Gets the absolute path for a given file path
* This will deliver the directory of the file as absolute path.
@ -181,6 +188,68 @@ std::string TL_PUBLIC current_dir ();
*/
bool TL_PUBLIC chdir (const std::string &path);
/**
* @brief Gets a temporary file path
*
* This function will make a temporary file with a unique name.
* The "domain" string is used as part of the file name as an disambiguator.
*
* The function reads $TMPDIR or $TMP to define the location of the temporary
* directory. On Linux, the default is /tmp.
*
* The file is created and it is the responsibility of the caller to remove
* the file.
*/
std::string TL_PUBLIC tmpfile (const std::string &domain = std::string ());
/**
* @brief A class wrapping a temporary file
*
* In the destructor of this class, the temporary file will be deleted again.
*/
class TL_PUBLIC TemporaryFile
{
public:
TemporaryFile (const std::string &domain = std::string ());
~TemporaryFile ();
const std::string &path () const
{
return m_path;
}
private:
std::string m_path;
};
/**
* @brief Gets a temporary folder path
*
* Similar to "tmpfile", but will create a new, empty folder. Again it is the
* reposibility of the caller to clean up.
*/
std::string TL_PUBLIC tmpdir (const std::string &domain = std::string ());
/**
* @brief A class wrapping a temporary directory
*
* In the destructor of this class, the temporary directory will be deleted again.
*/
class TL_PUBLIC TemporaryDirectory
{
public:
TemporaryDirectory (const std::string &domain = std::string ());
~TemporaryDirectory ();
const std::string &path () const
{
return m_path;
}
private:
std::string m_path;
};
/**
* @brief This function splits the path into it's components
* On Windows, the first component may be the drive prefix ("C:") or

421
src/tl/tl/tlGit.cc Normal file
View File

@ -0,0 +1,421 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2023 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#if defined(HAVE_GIT2)
#include "tlGit.h"
#include "tlFileUtils.h"
#include "tlProgress.h"
#include "tlStaticObjects.h"
#include "tlLog.h"
#include <git2.h>
#include <cstdio>
namespace tl
{
// ---------------------------------------------------------------
// Library initialization helper
namespace {
class InitHelper
{
public:
InitHelper ()
{
git_libgit2_init ();
}
~InitHelper ()
{
git_libgit2_shutdown ();
}
static void ensure_initialized ()
{
if (ms_init) {
return;
}
ms_init = new InitHelper ();
tl::StaticObjects::reg (&ms_init);
}
private:
static InitHelper *ms_init;
};
InitHelper *InitHelper::ms_init = 0;
}
// ---------------------------------------------------------------
// GitCollection implementation
GitObject::GitObject (const std::string &local_path)
: m_local_path (local_path), m_is_temp (false)
{
InitHelper::ensure_initialized ();
if (local_path.empty ()) {
m_local_path = tl::tmpdir ("git2klayout");
m_is_temp = true;
}
// ensures the directory is clean
if (! m_is_temp) {
if (! tl::rm_dir_recursive (m_local_path)) {
throw tl::Exception (tl::to_string (tr ("Unable to clean local Git repo path: %s")), m_local_path);
}
if (! tl::mkpath (m_local_path)) {
throw tl::Exception (tl::to_string (tr ("Unable to regenerate local Git repo path: %s")), m_local_path);
}
}
}
GitObject::~GitObject ()
{
if (m_is_temp) {
tl::rm_dir_recursive (m_local_path);
}
}
static int
#if LIBGIT2_VER_MAJOR >= 1
fetch_progress (const git_indexer_progress *stats, void *payload)
#else
fetch_progress (const git_transfer_progress *stats, void *payload)
#endif
{
tl::RelativeProgress *progress = reinterpret_cast<tl::RelativeProgress *> (payload);
// first half of progress
size_t count = size_t (5000.0 * double (stats->received_objects) / double (std::max (1u, stats->total_objects)) + 1e-10);
try {
progress->set (count);
} catch (...) {
// TODO: stop
}
return 0;
}
static void
checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload)
{
tl::RelativeProgress *progress = reinterpret_cast<tl::RelativeProgress *> (payload);
// first half of progress
size_t count = size_t (5000.0 * double (cur) / double (std::max (size_t (1), tot)) + 1e-10);
try {
progress->set (count + 5000u);
} catch (...) {
// ignore cancel requests (TODO: how to stop?)
}
}
static void check (int error)
{
if (error != 0) {
#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28)
const git_error *err = git_error_last ();
#else
const git_error *err = giterr_last ();
#endif
throw tl::Exception (tl::to_string (tr ("Error cloning Git repo: %s")), (const char *) err->message);
}
}
static bool
ref_matches (const char *name, const std::string &ref)
{
if (!name) {
return false;
} else if (name == ref) {
return true;
} else if (name == "refs/heads/" + ref) {
return true;
} else if (name == "refs/tags/" + ref) {
return true;
} else {
return false;
}
}
namespace
{
class GitBuffer
{
public:
GitBuffer ()
{
m_buf = GIT_BUF_INIT_CONST (NULL, 0);
}
~GitBuffer ()
{
#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28)
git_buf_dispose (&m_buf);
#else
git_buf_free (&m_buf);
#endif
}
const char *c_str () const { return m_buf.ptr; }
git_buf *get () { return &m_buf; }
const git_buf *get () const { return &m_buf; }
private:
git_buf m_buf;
};
}
static void
checkout_branch (git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch)
{
GitBuffer remote_branch;
git_oid oid;
// if no branch is given, use the default branch
if (! branch) {
check (git_remote_default_branch (remote_branch.get (), remote));
branch = remote_branch.c_str ();
if (tl::verbosity () >= 10) {
tl::info << tr ("Git checkout: Using default branch for repository ") << git_remote_url (remote) << ": " << branch;
}
} else {
if (tl::verbosity () >= 10) {
tl::info << tr ("Git checkout: Checking out branch for repository ") << git_remote_url (remote) << ": " << branch;
}
}
// resolve the branch by using ls-remote:
size_t n = 0;
const git_remote_head **ls = NULL;
check (git_remote_ls (&ls, &n, remote));
if (tl::verbosity () >= 20) {
tl::info << "Git checkout: ls-remote on " << git_remote_url (remote) << ":";
}
bool found = false;
for (size_t i = 0; i < n; ++i) {
const git_remote_head *rh = ls[i];
if (tl::verbosity () >= 20) {
char oid_fmt [80];
git_oid_tostr (oid_fmt, sizeof (oid_fmt), &rh->oid);
tl::info << " " << rh->name << ": " << (const char *) oid_fmt;
}
if (ref_matches (rh->name, branch)) {
oid = rh->oid;
found = true;
}
}
if (! found) {
throw tl::Exception (tl::to_string (tr ("Git checkout - Unable to resolve reference name: ")) + branch);
}
if (tl::verbosity () >= 10) {
char oid_fmt [80];
git_oid_tostr (oid_fmt, sizeof (oid_fmt), &oid);
tl::info << tr ("Git checkout: resolving ") << branch << tr (" to ") << (const char *) oid_fmt;
}
check (git_repository_set_head_detached (repo, &oid));
check (git_checkout_head (repo, co_opts));
}
int
credentials_cb (git_credential ** /*out*/, const char * /*url*/, const char * /*username*/, unsigned int /*allowed_types*/, void *)
{
// no credentials aquired
git_error_set_str (GIT_ERROR_NONE, "anonymous access is supported only, but server requests credentials");
return GIT_EUSER;
}
void
GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback)
{
std::string url = org_url;
std::string filter = org_filter;
if (! subfolder.empty ()) {
if (filter.empty ()) {
filter = subfolder + "/**";
} else {
filter = subfolder + "/" + filter;
}
}
// TODO: use callback, timeout?
tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/);
// build checkout options
git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT;
checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE;
const char *paths_cstr[1];
paths_cstr[0] = filter.c_str ();
if (! filter.empty ()) {
checkout_opts.paths.count = 1;
checkout_opts.paths.strings = (char **) &paths_cstr;
}
checkout_opts.progress_cb = &checkout_progress;
checkout_opts.progress_payload = (void *) &progress;
// build fetch options
git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT;
fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_AUTO;
#if LIBGIT2_VER_MAJOR > 1 || (LIBGIT2_VER_MAJOR == 1 && LIBGIT2_VER_MINOR >= 7)
fetch_opts.depth = 1; // shallow (single commit)
#endif
fetch_opts.callbacks.transfer_progress = &fetch_progress;
fetch_opts.callbacks.credentials = &credentials_cb;
fetch_opts.callbacks.payload = (void *) &progress;
// build refspecs in case they are needed
char *refs[] = { (char *) branch.c_str () };
git_strarray refspecs;
refspecs.count = 1;
refspecs.strings = refs;
git_strarray *refspecs_p = branch.empty () ? NULL : &refspecs;
// Make repository
git_repository *cloned_repo = NULL;
git_remote *remote = NULL;
try {
check (git_repository_init (&cloned_repo, m_local_path.c_str (), 0));
check (git_remote_create (&remote, cloned_repo, "download", url.c_str ()));
// actually fetch
if (tl::verbosity () >= 10) {
tl::info << tr ("Fetching Git repo from ") << git_remote_url (remote) << " ...";
}
check (git_remote_fetch (remote, refspecs_p, &fetch_opts, NULL));
// checkout
checkout_branch (cloned_repo, remote, &checkout_opts, branch.empty () ? 0 : branch.c_str ());
// free the repo and remote
git_repository_free (cloned_repo);
git_remote_free (remote);
// get rid of ".git" - we do not need it anymore
tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git"));
} catch (...) {
// free the repo in the error case
if (cloned_repo != NULL) {
git_repository_free (cloned_repo);
}
if (remote != NULL) {
git_remote_free (remote);
}
throw;
}
// pull subfolder files to target path level
if (! subfolder.empty ()) {
std::string pp = tl::combine_path (m_local_path, subfolder);
if (! tl::is_dir (pp)) {
throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to fetch subdirectory: ")) + pp);
}
// rename the source to a temporary folder so we don't overwrite the source folder with something from within
std::string tmp_dir = "__temp__";
for (unsigned int i = 0; ; ++i) {
if (! tl::file_exists (tl::combine_path (m_local_path, tmp_dir + tl::to_string (i)))) {
tmp_dir += tl::to_string (i);
break;
}
}
auto pc = tl::split (subfolder, "/");
if (! tl::rename_file (tl::combine_path (m_local_path, pc.front ()), tmp_dir)) {
throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to rename temp folder")));
}
pc.front () = tmp_dir;
if (! tl::mv_dir_recursive (tl::combine_path (m_local_path, tl::join (pc, "/")), m_local_path)) {
throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to move subdir components")));
}
}
}
bool
GitObject::download (const std::string &url, const std::string &target, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback)
{
try {
GitObject obj (target);
obj.read (url, std::string (), subfolder, branch, timeout, callback);
return true;
} catch (tl::Exception &ex) {
tl::error << tl::sprintf (tl::to_string (tr ("Error downloading Git repo from %s (subdir '%s', ref '%s')")), url, subfolder, branch);
return false;
}
}
tl::InputStream *
GitObject::download_item (const std::string &url, const std::string &file, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback)
{
GitObject obj;
obj.read (url, file, subfolder, branch, timeout, callback);
// extract the file and return a memory blob, so we can delete the temp folder
tl::InputStream file_stream (tl::combine_path (obj.local_path (), file));
std::string data = file_stream.read_all ();
char *data_copy = new char [data.size ()];
memcpy (data_copy, data.c_str (), data.size ());
return new tl::InputStream (new tl::InputMemoryStream (data_copy, data.size (), true));
}
}
#endif

111
src/tl/tl/tlGit.h Normal file
View File

@ -0,0 +1,111 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2023 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef HDR_tlGit
#define HDR_tlGit
#include "tlCommon.h"
#include "tlStream.h"
#include <string>
#include <vector>
#if !defined(HAVE_GIT2)
# error "tlGit.h can only be used with libgit2 enabled"
#endif
namespace tl
{
class InputHttpStreamCallback;
/**
* @brief Represents an object from a Git URL
* This object can be a file or collection
*/
class TL_PUBLIC GitObject
{
public:
/**
* @brief Open a stream with the given URL
*
* The local_path is the path where to store the files.
* If empty, a temporary folder is created and destroyed once the "GitObject" goes out of scope.
*/
GitObject (const std::string &local_path = std::string ());
/**
* @brief Destructor
*/
~GitObject ();
/**
* @brief Populates the collection from the given URL
*
* "filter" can be a top-level file to download. If filter is non-empty,
* sparse mode is chosen.
*/
void read (const std::string &url, const std::string &filter, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);
/**
* @brief Downloads the collection or file with the given URL
*
* This method will download the Git object from url to the file path
* given in "target".
*
* For file download, the target must be the path of the target file.
* For collection download, the target must be a directory path. In this
* case, the target directory must exist already.
*
* Sub-directories are created if required.
*
* This method returns false if the directory structure could
* not be obtained or downloading of one file failed.
*
* "branch" is the remote ref to use. This can be a branch name, a tag name,
* a remote ref such as "refs/heads/master" or a symbolic name such as "HEAD".
*/
static bool download (const std::string &url, const std::string &target, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);
/**
* @brief Gets a stream object for downloading the single item of the given URL
*
* The file needs to be a top-level object.
* The stream object returned needs to be deleted by the caller.
*/
static tl::InputStream *download_item (const std::string &url, const std::string &file, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);
private:
std::string m_local_path;
bool m_is_temp;
const std::string &local_path () const
{
return m_local_path;
}
};
}
#endif

2
src/tl/tl/tlWebDAV.h
View File

@ -145,7 +145,7 @@ public:
*
* Sub-directories are created if required.
*
* This method throws an exception if the directory structure could
* This method returns false if the directory structure could
* not be obtained or downloading of one file failed.
*/
static bool download (const std::string &url, const std::string &target, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0);

143
src/tl/unit_tests/tlFileUtilsTests.cc
View File

@ -285,6 +285,58 @@ TEST (3_NOQT)
}
}
TEST (4_mv_dir)
{
std::string tmp_dir = tl::absolute_file_path (tmp_file ());
std::string adir = tl::combine_path (tmp_dir, "a");
std::string b1dir = tl::combine_path (adir, "b1");
tl::mkpath (b1dir);
EXPECT_EQ (tl::file_exists (b1dir), true);
std::string b2dir = tl::combine_path (adir, "b2");
tl::mkpath (b2dir);
EXPECT_EQ (tl::file_exists (b1dir), true);
{
tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "x")));
os << "hello, world!\n";
}
{
tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "y")));
os << "hello, world II!\n";
}
std::string acopydir = tl::combine_path (tmp_dir, "acopy");
tl::rm_dir_recursive (acopydir);
tl::mkpath (acopydir);
tl::mv_dir_recursive (adir, acopydir);
EXPECT_EQ (tl::file_exists (acopydir), true);
EXPECT_EQ (tl::file_exists (adir), false);
std::string b1copydir = tl::combine_path (acopydir, "b1");
EXPECT_EQ (tl::file_exists (b1copydir), true);
std::string b2copydir = tl::combine_path (acopydir, "b2");
EXPECT_EQ (tl::file_exists (b2copydir), true);
{
std::string xfile = tl::combine_path (b2copydir, "x");
EXPECT_EQ (tl::file_exists (xfile), true);
tl::InputStream is (xfile);
EXPECT_EQ (is.read_all (), "hello, world!\n");
}
{
std::string yfile = tl::combine_path (b2copydir, "y");
EXPECT_EQ (tl::file_exists (yfile), true);
tl::InputStream is (yfile);
EXPECT_EQ (is.read_all (), "hello, world II!\n");
}
}
// Secret mode switchers for testing
namespace tl
{
@ -797,6 +849,13 @@ TEST (18)
tl::InputStream is (zfile);
EXPECT_EQ (is.read_all (), "hello, world!\n");
}
// rename directory
tl::rename_file (tl::combine_path (tp, "dir"), "dirx");
EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dir")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dirx")), true);
EXPECT_EQ (tl::is_dir (tl::combine_path (tp, "dirx")), true);
}
// get_home_path
@ -816,3 +875,87 @@ TEST (20)
EXPECT_EQ (tl::absolute_file_path ("~"), tl::get_home_path ());
EXPECT_EQ (tl::absolute_file_path (tl::combine_path ("~", "test")), tl::combine_path (tl::get_home_path (), "test"));
}
// tmpfile
TEST (21)
{
std::string p = tl::tmpfile ("tl_tests");
EXPECT_EQ (tl::file_exists (p), true);
std::ofstream os (p);
os << "A test";
os.close ();
{
tl::InputStream is (p);
EXPECT_EQ (is.read_all (), "A test");
}
EXPECT_EQ (tl::rm_file (p), true);
EXPECT_EQ (tl::file_exists (p), false);
}
// TemporaryFile
TEST (22)
{
std::string p;
{
tl::TemporaryFile tf ("tl_tests");
p = tf.path ();
EXPECT_EQ (tl::file_exists (tf.path ()), true);
std::ofstream os (tf.path ());
os << "A test";
os.close ();
tl::InputStream is (tf.path ());
EXPECT_EQ (is.read_all (), "A test");
}
EXPECT_EQ (tl::file_exists (p), false);
}
// tmpdir
TEST (23)
{
std::string p = tl::tmpdir ("tl_tests");
EXPECT_EQ (tl::file_exists (p), true);
EXPECT_EQ (tl::is_dir (p), true);
std::ofstream os (tl::combine_path (p, "test"));
os << "A test";
os.close ();
{
tl::InputStream is (tl::combine_path (p, "test"));
EXPECT_EQ (is.read_all (), "A test");
}
EXPECT_EQ (tl::rm_dir_recursive (p), true);
EXPECT_EQ (tl::file_exists (p), false);
}
// TemporaryDirectory object
TEST (24)
{
std::string p;
{
tl::TemporaryDirectory tmpdir ("tl_tests");
p = tmpdir.path ();
EXPECT_EQ (tl::file_exists (p), true);
EXPECT_EQ (tl::is_dir (p), true);
std::ofstream os (tl::combine_path (p, "test"));
os << "A test";
os.close ();
tl::InputStream is (tl::combine_path (p, "test"));
EXPECT_EQ (is.read_all (), "A test");
}
EXPECT_EQ (tl::file_exists (p), false);
}

218
src/tl/unit_tests/tlGitTests.cc Normal file
View File

@ -0,0 +1,218 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2023 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#if defined(HAVE_GIT2)
#include "tlGit.h"
#include "tlUnitTest.h"
#include "tlFileUtils.h"
static std::string test_url ("https://github.com/klayout/klayout_git_test.git");
static std::string test_url_invalid ("https://github.com/klayout/doesnotexist.git");
TEST(1_plain)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string (), std::string (), std::string ());
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true);
}
TEST(2_subdir)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string (), std::string ("src"), std::string ());
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros/xsection.lym")), true);
}
TEST(3_subdir_as_filter)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string ("src/**"), std::string (), std::string ());
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true);
}
TEST(4_single_file)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string ("LICENSE"), std::string (), std::string ());
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src")), false);
}
TEST(5_single_file_from_subdir)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ());
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false);
tl::InputStream file (tl::combine_path (path, "grain.xml"));
tl::TextInputStream grain (file);
bool found = false;
while (! grain.at_end () && ! found) {
std::string line = grain.get_line ();
if (line.find ("<version>1.7</version>") != std::string::npos) {
found = true;
}
}
EXPECT_EQ (found, true);
}
TEST(6_branch)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("wip"));
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false);
tl::InputStream file (tl::combine_path (path, "grain.xml"));
tl::TextInputStream grain (file);
bool found = false;
while (! grain.at_end () && ! found) {
std::string line = grain.get_line ();
if (line.find ("<version>1.4</version>") != std::string::npos) {
found = true;
}
}
EXPECT_EQ (found, true);
}
TEST(7_tag)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("1.2"));
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false);
tl::InputStream file (tl::combine_path (path, "grain.xml"));
tl::TextInputStream grain (file);
bool found = false;
while (! grain.at_end () && ! found) {
std::string line = grain.get_line ();
if (line.find ("<version>1.2</version>") != std::string::npos) {
found = true;
}
}
EXPECT_EQ (found, true);
}
TEST(8_refspec)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("refs/tags/1.5"));
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false);
tl::InputStream file (tl::combine_path (path, "grain.xml"));
tl::TextInputStream grain (file);
bool found = false;
while (! grain.at_end () && ! found) {
std::string line = grain.get_line ();
if (line.find ("<version>1.5</version>") != std::string::npos) {
found = true;
}
}
EXPECT_EQ (found, true);
}
TEST(9_HEAD)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("HEAD"));
EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true);
EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false);
tl::InputStream file (tl::combine_path (path, "grain.xml"));
tl::TextInputStream grain (file);
bool found = false;
while (! grain.at_end () && ! found) {
std::string line = grain.get_line ();
if (line.find ("<version>1.7</version>") != std::string::npos) {
found = true;
}
}
EXPECT_EQ (found, true);
}
TEST(10_invalid_branch)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
try {
repo.read (test_url, std::string (), std::string (), std::string ("brxxx"));
EXPECT_EQ (true, false);
} catch (tl::Exception &ex) {
EXPECT_EQ (ex.msg (), "Git checkout - Unable to resolve reference name: brxxx");
}
}
TEST(11_invalid_url)
{
std::string path = tl::TestBase::tmp_file ("repo");
tl::GitObject repo (path);
try {
repo.read (test_url_invalid, std::string (), std::string (), std::string ("brxxx"));
EXPECT_EQ (true, false);
} catch (tl::Exception &ex) {
EXPECT_EQ (ex.msg (), "Error cloning Git repo: anonymous access is supported only, but server requests credentials");
}
}
#endif

1
src/tl/unit_tests/unit_tests.pro
View File

@ -20,6 +20,7 @@ SOURCES = \
tlExpressionTests.cc \
tlFileSystemWatcherTests.cc \
tlFileUtilsTests.cc \
tlGitTests.cc \
tlHttpStreamTests.cc \
tlIncludeTests.cc \
tlInt128SupportTests.cc \

4
src/unit_tests/unit_test_main.cc
View File

@ -433,6 +433,8 @@ run_tests (const std::vector<tl::TestBase *> &selected_tests, bool editable, boo
static int
main_cont (int &argc, char **argv)
{
ut::TestConsole console (stdout);
std::unique_ptr<rba::RubyInterpreter> ruby_interpreter;
std::unique_ptr<pya::PythonInterpreter> python_interpreter;
@ -452,8 +454,6 @@ main_cont (int &argc, char **argv)
int result = 0;
ut::TestConsole console (stdout);
try {
pya::PythonInterpreter::initialize ();