diff --git a/src/lay/SaltManagerInstallConfirmationDialog.ui b/src/lay/SaltManagerInstallConfirmationDialog.ui new file mode 100644 index 000000000..0920a619b --- /dev/null +++ b/src/lay/SaltManagerInstallConfirmationDialog.ui @@ -0,0 +1,156 @@ + + + SaltManagerInstallConfirmationDialog + + + + 0 + 0 + 495 + 478 + + + + Ready for Installation + + + + + + The following packages are now ready for installation or update: + + + true + + + + + + + + Package + + + + + Action + + + + + Version + + + + + Download link + + + + + + + + Press "Ok" to install or update these packages or "Cancel" to abort. + + + + + + + Qt::Vertical + + + QSizePolicy::Fixed + + + + 20 + 6 + + + + + + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + :/add.png:/add.png + + + New + + + New package + + + + + + :/clear.png:/clear.png + + + Delete + + + Delete package + + + + + + :/import.png:/import.png + + + Import + + + Import package + + + + + + + + + buttonBox + accepted() + SaltManagerInstallConfirmationDialog + accept() + + + 273 + 431 + + + 276 + 448 + + + + + buttonBox + rejected() + SaltManagerInstallConfirmationDialog + reject() + + + 351 + 426 + + + 363 + 445 + + + + + diff --git a/src/lay/lay.pro b/src/lay/lay.pro index 7d1bff20d..0daa9efc7 100644 --- a/src/lay/lay.pro +++ b/src/lay/lay.pro @@ -101,7 +101,8 @@ FORMS = \ MainConfigPage7.ui \ SaltManagerDialog.ui \ SaltGrainPropertiesDialog.ui \ - SaltGrainTemplateSelectionDialog.ui + SaltGrainTemplateSelectionDialog.ui \ + SaltManagerInstallConfirmationDialog.ui SOURCES = \ gsiDeclLayApplication.cc \ diff --git a/src/lay/laySalt.cc b/src/lay/laySalt.cc index c16bc8e56..2812d2095 100644 --- a/src/lay/laySalt.cc +++ b/src/lay/laySalt.cc @@ -21,11 +21,11 @@ */ #include "laySalt.h" -#include "laySaltDownloadManager.h" #include "tlString.h" #include "tlFileUtils.h" #include "tlLog.h" #include "tlInternational.h" +#include "tlWebDAV.h" #include #include @@ -307,12 +307,23 @@ public: } bool -Salt::create_grain (const SaltGrain &templ, SaltGrain &target, SaltDownloadManager *download_manager) +Salt::create_grain (const SaltGrain &templ, SaltGrain &target) { tl_assert (!m_root.is_empty ()); const SaltGrains *coll = m_root.begin_collections ().operator-> (); + if (target.name ().empty ()) { + target.set_name (templ.name ()); + } + + if (target.path ().empty ()) { + lay::SaltGrain *g = grain_by_name (target.name ()); + if (g) { + target.set_path (g->path ()); + } + } + std::string path = target.path (); if (! path.empty ()) { coll = 0; @@ -383,11 +394,11 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, SaltDownloadManag } else if (! templ.url ().empty ()) { - tl_assert (download_manager != 0); - // otherwise download from the URL tl::info << QObject::tr ("Downloading package from '%1' to '%2' ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); - res = download_manager->download (templ.url (), target.path ()); + res = tl::WebDAVObject::download (templ.url (), target.path ()); + + target.set_url (templ.url ()); } diff --git a/src/lay/laySalt.h b/src/lay/laySalt.h index ff16c0280..d238476ae 100644 --- a/src/lay/laySalt.h +++ b/src/lay/laySalt.h @@ -34,8 +34,6 @@ namespace lay { -class SaltDownloadManager; - /** * @brief The global salt (package manager) object * This object can be configured to represent a couple of locations. @@ -123,6 +121,14 @@ public: */ SaltGrain *grain_by_name (const std::string &name); + /** + * @brief Gets the grain with the given name (const version) + */ + const SaltGrain *grain_by_name (const std::string &name) const + { + return const_cast (this)->grain_by_name (name); + } + /** * @brief Loads the salt from a "salt mine" file */ @@ -165,11 +171,11 @@ public: * all files related to this grain. It will copy the download URL from the template into the * new grain, so updates will come from the original location. * - * The target's name must be set. If a specific target location is desired, the target's - * path must be set too. - * - * This method refuses to overwrite existing grains, so an update needs to be performed by first - * deleting the grain and then re-installing it. + * If the target's name is not set, it will be taken from the template. + * If the target's path is not set and a grain with the given name already exists in + * the package, the path is taken from that grain. + * If no target path is set and no grain with this name exists yet, a new path will + * be constructed using the first location in the salt. * * The target grain will be updated with the installation information. If the target grain * contains an installation path prior to the installation, this path will be used for the @@ -177,7 +183,7 @@ public: * * Returns true, if the package could be created successfully. */ - bool create_grain (const SaltGrain &templ, SaltGrain &target, SaltDownloadManager *download_manager = 0); + bool create_grain (const SaltGrain &templ, SaltGrain &target); signals: /** diff --git a/src/lay/laySaltDownloadManager.cc b/src/lay/laySaltDownloadManager.cc index f6970b851..c3e587551 100644 --- a/src/lay/laySaltDownloadManager.cc +++ b/src/lay/laySaltDownloadManager.cc @@ -21,20 +21,189 @@ */ #include "laySaltDownloadManager.h" +#include "laySalt.h" +#include "tlFileUtils.h" +#include "tlWebDAV.h" + +#include "ui_SaltManagerInstallConfirmationDialog.h" + +#include namespace lay { +// ---------------------------------------------------------------------------------- + +class ConfirmationDialog + : public QDialog, private Ui::SaltManagerInstallConfirmationDialog +{ +public: + ConfirmationDialog (QWidget *parent) + : QDialog (parent) + { + Ui::SaltManagerInstallConfirmationDialog::setupUi (this); + } + + void add_info (const std::string &name, bool update, const std::string &version, const std::string &url) + { + QTreeWidgetItem *item = new QTreeWidgetItem (list); + item->setText (0, tl::to_qstring (name)); + item->setText (1, update ? tr ("UPDATE") : tr ("INSTALL")); + item->setText (2, tl::to_qstring (version)); + item->setText (3, tl::to_qstring (url)); + } +}; + +// ---------------------------------------------------------------------------------- + SaltDownloadManager::SaltDownloadManager () { // .. nothing yet .. } -bool -SaltDownloadManager::download (const std::string &url, const std::string &target_dir) +void +SaltDownloadManager::register_download (const std::string &name, const std::string &url, const std::string &version) { - // @@@ + m_registry.insert (std::make_pair (name, Descriptor (url, version))); +} + +void +SaltDownloadManager::compute_dependencies (const lay::Salt &salt, const lay::Salt &salt_mine) +{ + tl::AbsoluteProgress progress (tl::to_string (QObject::tr ("Computing package dependencies .."))); + + while (needs_iteration ()) { + + fetch_missing (salt_mine, progress); + + std::map registry = m_registry; + for (std::map::const_iterator p = registry.begin (); p != registry.end (); ++p) { + + for (std::vector::const_iterator d = p->second.grain.dependencies ().begin (); d != p->second.grain.dependencies ().end (); ++d) { + + std::map::iterator r = m_registry.find (d->name); + if (r != m_registry.end ()) { + + if (SaltGrain::compare_versions (r->second.version, d->version) < 0) { + + // Grain is present, but too old -> update version and reload in the next iteration + r->second.downloaded = false; + r->second.version = d->version; + r->second.url = d->url; + r->second.downloaded = false; + + } + + } else { + + const SaltGrain *g = salt.grain_by_name (d->name); + if (g) { + + // Grain is installed already, but too old -> register for update + if (SaltGrain::compare_versions (g->version (), d->version) < 0) { + register_download (d->name, d->url, d->version); + } + + } else { + register_download (d->name, d->url, d->version); + } + + } + + } + + } + + } +} + +bool +SaltDownloadManager::needs_iteration () +{ + for (std::map::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) { + if (! p->second.downloaded) { + return true; + } + } return false; } +void +SaltDownloadManager::fetch_missing (const lay::Salt &salt_mine, tl::AbsoluteProgress &progress) +{ + for (std::map::iterator p = m_registry.begin (); p != m_registry.end (); ++p) { + + if (! p->second.downloaded) { + + ++progress; + + // If no URL is given, utilize the salt mine to fetch it + if (p->second.url.empty ()) { + const lay::SaltGrain *g = salt_mine.grain_by_name (p->first); + if (SaltGrain::compare_versions (g->version (), p->second.version) < 0) { + throw tl::Exception (tl::to_string (QObject::tr ("Package '%1': package in repository is too old (%2) to satisfy requirements (%3)").arg (tl::to_qstring (p->first)).arg (tl::to_qstring (g->version ())).arg (tl::to_qstring (p->second.version)))); + } + p->second.version = g->version (); + p->second.url = g->url (); + } + + p->second.grain = SaltGrain::from_url (p->second.url); + p->second.downloaded = true; + + } + + } +} + +bool +SaltDownloadManager::show_confirmation_dialog (QWidget *parent, const lay::Salt &salt) +{ + lay::ConfirmationDialog dialog (parent); + + // First the packages to update + for (std::map::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) { + const lay::SaltGrain *g = salt.grain_by_name (p->first); + if (g) { + dialog.add_info (p->first, true, g->version () + "->" + p->second.version, p->second.url); + } + } + + // Then the packages to install + for (std::map::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) { + const lay::SaltGrain *g = salt.grain_by_name (p->first); + if (!g) { + dialog.add_info (p->first, false, p->second.version, p->second.url); + } + } + + return dialog.exec (); +} + +bool +SaltDownloadManager::execute (lay::Salt &salt) +{ + bool result = true; + + tl::RelativeProgress progress (tl::to_string (QObject::tr ("Downloading packages")), m_registry.size (), 1); + + for (std::map::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) { + + lay::SaltGrain target; + target.set_name (p->first); + lay::SaltGrain *g = salt.grain_by_name (p->first); + if (g) { + target.set_path (g->path ()); + } + + if (! salt.create_grain (p->second.grain, target)) { + result = false; + } + + ++progress; + + } + + return result; +} + } diff --git a/src/lay/laySaltDownloadManager.h b/src/lay/laySaltDownloadManager.h index 141df238a..89fd74620 100644 --- a/src/lay/laySaltDownloadManager.h +++ b/src/lay/laySaltDownloadManager.h @@ -24,17 +24,27 @@ #define HDR_laySaltDownloadManager #include "layCommon.h" +#include "laySaltGrain.h" +#include "tlProgress.h" #include #include +#include namespace lay { +class Salt; + /** * @brief The download manager + * * This class is responsible for handling the downloads for - * grains. + * grains. The basic sequence is: + * + "register_download" (multiple times) to register the packages intended for download + * + "compute_dependencies" to determine all related packages + * + (optional) "show_confirmation_dialog" + * + "execute" to actually execute the downloads */ class LAY_PUBLIC SaltDownloadManager : public QObject @@ -48,11 +58,58 @@ public: SaltDownloadManager (); /** - * @brief Downloads the files from the given URL to the given target location - * The target directory needs to exist. - * Returns true, if the download was successful, false otherwise. + * @brief Registers an URL (with version) for download in the given target directory + * + * The target directory can be empty. In this case, the downloader will pick an approriate one. */ - bool download (const std::string &url, const std::string &target_dir); + void register_download (const std::string &name, const std::string &url, const std::string &version); + + /** + * @brief Computes the dependencies after all required packages have been registered + * + * This method will compute the dependencies. Packages not present in the list of + * packages ("salt" argument), will be scheduled for download too. Dependency packages + * are looked up in "salt_mine" if no download URL is given. + */ + void compute_dependencies (const lay::Salt &salt, const Salt &salt_mine); + + /** + * @brief Presents a dialog showing the packages scheduled for download + * + * This method requires all dependencies to be computed. It will return false + * if the dialog is not confirmed. + * + * "salt" needs to be the currently installed packages so the dialog can + * indicate which packages will be updated. + */ + bool show_confirmation_dialog (QWidget *parent, const lay::Salt &salt); + + /** + * @brief Actually execute the downloads + * + * This method will return false if anything goes wrong. + * Failed packages will be removed entirely after they have been listed in + * an error dialog. + */ + bool execute (lay::Salt &salt); + +private: + struct Descriptor + { + Descriptor (const std::string &_url, const std::string &_version) + : url (_url), version (_version), downloaded (false) + { } + + std::string url; + std::string version; + bool downloaded; + lay::SaltGrain grain; + }; + + std::map m_registry; + + bool needs_iteration (); + void fetch_missing (const lay::Salt &salt_mine, tl::AbsoluteProgress &progress); }; } diff --git a/src/lay/laySaltManagerDialog.cc b/src/lay/laySaltManagerDialog.cc index 57ce3fbf5..978dc9c46 100644 --- a/src/lay/laySaltManagerDialog.cc +++ b/src/lay/laySaltManagerDialog.cc @@ -23,6 +23,7 @@ #include "laySaltManagerDialog.h" #include "laySaltModel.h" #include "laySaltGrainPropertiesDialog.h" +#include "laySaltDownloadManager.h" #include "laySalt.h" #include "ui_SaltGrainTemplateSelectionDialog.h" #include "tlString.h" @@ -151,6 +152,7 @@ SaltManagerDialog::SaltManagerDialog (QWidget *parent) connect (edit_button, SIGNAL (clicked ()), this, SLOT (edit_properties ())); connect (create_button, SIGNAL (clicked ()), this, SLOT (create_grain ())); connect (delete_button, SIGNAL (clicked ()), this, SLOT (delete_grain ())); + connect (apply_button, SIGNAL (clicked ()), this, SLOT (apply ())); mp_salt = get_salt (); mp_salt_mine = get_salt_mine (); @@ -258,6 +260,42 @@ SaltManagerDialog::mark_clicked () model->set_marked (g->name (), !model->is_marked (g->name ())); } +void +SaltManagerDialog::apply () +{ +BEGIN_PROTECTED + + lay::SaltDownloadManager manager; + + bool any = false; + + // fetch all marked grains and register for download + SaltModel *model = dynamic_cast (salt_mine_view->model ()); + if (model) { + for (int i = model->rowCount (QModelIndex ()); i > 0; ) { + --i; + QModelIndex index = model->index (i, 0, QModelIndex ()); + SaltGrain *g = model->grain_from_index (index); + if (g && model->is_marked (g->name ())) { + manager.register_download (g->name (), g->url (), g->version ()); + any = true; + } + } + } + + if (! any) { + throw tl::Exception (tl::to_string (tr ("No packages marked for installation or update"))); + } + + manager.compute_dependencies (*mp_salt, *mp_salt_mine); + + if (manager.show_confirmation_dialog (this, *mp_salt)) { + manager.execute (*mp_salt); + } + +END_PROTECTED +} + void SaltManagerDialog::edit_properties () { diff --git a/src/lay/laySaltManagerDialog.h b/src/lay/laySaltManagerDialog.h index cbbfac0a4..c6e4ff893 100644 --- a/src/lay/laySaltManagerDialog.h +++ b/src/lay/laySaltManagerDialog.h @@ -95,6 +95,11 @@ private slots: */ void mode_changed (); + /** + * @brief Called when the "apply" button is clicked + */ + void apply (); + /** * @brief Called when one search text changed */ diff --git a/src/tl/tl.pro b/src/tl/tl.pro index 39a9e5713..0a427e5e2 100644 --- a/src/tl/tl.pro +++ b/src/tl/tl.pro @@ -40,7 +40,8 @@ SOURCES = \ tlXMLParser.cc \ tlXMLWriter.cc \ tlFileSystemWatcher.cc \ - tlFileUtils.cc + tlFileUtils.cc \ + tlWebDAV.cc HEADERS = \ tlAlgorithm.h \ @@ -83,7 +84,8 @@ HEADERS = \ tlXMLWriter.h \ tlFileSystemWatcher.h \ tlCommon.h \ - tlFileUtils.h + tlFileUtils.h \ + tlWebDAV.h INCLUDEPATH = DEPENDPATH = diff --git a/src/tl/tlHttpStream.cc b/src/tl/tlHttpStream.cc index 216e31c76..d91bfb905 100644 --- a/src/tl/tlHttpStream.cc +++ b/src/tl/tlHttpStream.cc @@ -87,7 +87,6 @@ InputHttpStream::InputHttpStream (const std::string &url) connect (s_network_manager, SIGNAL (finished (QNetworkReply *)), this, SLOT (finished (QNetworkReply *))); connect (s_network_manager, SIGNAL (authenticationRequired (QNetworkReply *, QAuthenticator *)), this, SLOT (authenticationRequired (QNetworkReply *, QAuthenticator *))); connect (s_network_manager, SIGNAL (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)), this, SLOT (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *))); - issue_request (QUrl (tl::to_qstring (url))); mp_reply = 0; } @@ -115,6 +114,12 @@ InputHttpStream::set_data (const char *data, size_t n) m_data = QByteArray (data, int (n)); } +void +InputHttpStream::add_header (const std::string &name, const std::string &value) +{ + m_headers.insert (std::make_pair (name, value)); +} + void InputHttpStream::authenticationRequired (QNetworkReply *reply, QAuthenticator *auth) { @@ -148,19 +153,26 @@ InputHttpStream::issue_request (const QUrl &url) delete mp_buffer; mp_buffer = 0; + QNetworkRequest request (url); + for (std::map::const_iterator h = m_headers.begin (); h != m_headers.end (); ++h) { + request.setRawHeader (QByteArray (h->first.c_str ()), QByteArray (h->second.c_str ())); + } if (m_data.isEmpty ()) { - s_network_manager->sendCustomRequest (QNetworkRequest (url), m_request); + s_network_manager->sendCustomRequest (request, m_request); } else { mp_buffer = new QBuffer (&m_data); - s_network_manager->sendCustomRequest (QNetworkRequest (url), m_request, mp_buffer); + s_network_manager->sendCustomRequest (request, m_request, mp_buffer); } } size_t InputHttpStream::read (char *b, size_t n) { + if (mp_reply == 0) { + issue_request (QUrl (tl::to_qstring (m_url))); + } while (mp_reply == 0) { - QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents); + QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents, 100); } if (mp_reply->error () != QNetworkReply::NoError) { diff --git a/src/tl/tlHttpStream.h b/src/tl/tlHttpStream.h index 7f20c0aad..acd3b8635 100644 --- a/src/tl/tlHttpStream.h +++ b/src/tl/tlHttpStream.h @@ -90,6 +90,11 @@ public: */ void set_data (const char *data, size_t n); + /** + * @brief Sets a header field + */ + void add_header (const std::string &name, const std::string &value); + /** * @brief Read from the stream * Implements the basic read method. @@ -121,6 +126,7 @@ private: QByteArray m_request; QByteArray m_data; QBuffer *mp_buffer; + std::map m_headers; void issue_request (const QUrl &url); }; diff --git a/src/tl/tlWebDAV.cc b/src/tl/tlWebDAV.cc new file mode 100644 index 000000000..19c2a004e --- /dev/null +++ b/src/tl/tlWebDAV.cc @@ -0,0 +1,316 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2017 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlWebDAV.h" +#include "tlXMLParser.h" +#include "tlHttpStream.h" +#include "tlStream.h" +#include "tlInternational.h" +#include "tlProgress.h" +#include "tlLog.h" + +#include +#include + +namespace tl +{ + +// --------------------------------------------------------------- +// WebDAVCollection implementation + +WebDAVObject::WebDAVObject () +{ + // .. nothing yet .. +} + +namespace +{ + +/** + * @brief A dummy "DOM" for the WebDAV reply + */ +struct ResourceType +{ + ResourceType () : is_collection (false) { } + + const std::string &collection () const + { + static std::string empty; + return empty; + } + + void set_collection (const std::string &) + { + is_collection = true; + } + + bool is_collection; +}; + +/** + * @brief A dummy "DOM" for the WebDAV reply + */ +struct Prop +{ + ResourceType resourcetype; +}; + +/** + * @brief A dummy "DOM" for the WebDAV reply + */ +struct PropStat +{ + std::string status; + Prop prop; +}; + +/** + * @brief A dummy "DOM" for the WebDAV reply + */ +struct Response +{ + std::string href; + PropStat propstat; +}; + +/** + * @brief A dummy "DOM" for the WebDAV reply + */ +struct MultiStatus +{ + typedef std::list container; + typedef container::const_iterator iterator; + + iterator begin () const { return responses.begin (); } + iterator end () const { return responses.end (); } + void add (const Response &r) { responses.push_back (r); } + + container responses; +}; + +} + +tl::XMLStruct xml_struct ("multistatus", + tl::make_element (&MultiStatus::begin, &MultiStatus::end, &MultiStatus::add, "response", + tl::make_member (&Response::href, "href") + + tl::make_element (&Response::propstat, "propstat", + tl::make_member (&PropStat::status, "status") + + tl::make_element (&PropStat::prop, "prop", + tl::make_element (&Prop::resourcetype, "resourcetype", + tl::make_member (&ResourceType::collection, &ResourceType::set_collection, "collection") + ) + ) + ) + ) +); + +static std::string item_name (const QString &path1, const QString &path2) +{ + QStringList sl1 = path1.split (QChar ('/')); + if (! sl1.empty () && sl1.back ().isEmpty ()) { + sl1.pop_back (); + } + + QStringList sl2 = path2.split (QChar ('/')); + if (! sl2.empty () && sl2.back ().isEmpty ()) { + sl2.pop_back (); + } + + int i = 0; + for ( ; i < sl1.length () && i < sl2.length (); ++i) { + if (sl1 [i] != sl2 [i]) { + throw tl::Exception (tl::to_string (QObject::tr ("Invalid WebDAV response: %1 is not a collection item of %2").arg (path2).arg (path1))); + } + } + if (i == sl2.length ()) { + return std::string (); + } else if (i + 1 == sl2.length ()) { + return tl::to_string (sl2[i]); + } else { + throw tl::Exception (tl::to_string (QObject::tr ("Invalid WebDAV response: %1 is not a collection sub-item of %2").arg (path2).arg (path1))); + } +} + +void +WebDAVObject::read (const std::string &url, int depth) +{ + QUrl base_url = QUrl (tl::to_qstring (url)); + + tl::InputHttpStream http (url); + http.add_header ("User-Agent", "SVN"); + http.add_header ("Depth", tl::to_string (depth)); + http.set_request ("PROPFIND"); + http.set_data (""); + + MultiStatus multistatus; + tl::InputStream stream (http); + tl::XMLStreamSource source (stream); + xml_struct.parse (source, multistatus); + + // TODO: check status .. + + m_items.clear (); + for (MultiStatus::iterator r = multistatus.begin (); r != multistatus.end (); ++r) { + + bool is_collection = r->propstat.prop.resourcetype.is_collection; + QUrl item_url = base_url.resolved (QUrl (tl::to_qstring (r->href))); + + std::string n = item_name (base_url.path (), item_url.path ()); + std::string item_url_string = tl::to_string (item_url.toString ()); + + if (! n.empty ()) { + m_items.push_back (WebDAVItem (is_collection, item_url_string, n)); + } else { + m_is_collection = is_collection; + m_url = item_url_string; + } + + } +} + +namespace +{ + +struct DownloadItem +{ + DownloadItem (const std::string &u, const std::string &p) + { + url = u; + path = p; + } + + std::string url; + std::string path; +}; + +} + +static +void fetch_download_items (const std::string &url, const std::string &target, std::list &items, tl::AbsoluteProgress &progress) +{ + ++progress; + + WebDAVObject object; + object.read (url, 1); + + if (object.is_collection ()) { + + QDir dir (tl::to_qstring (target)); + if (! dir.exists ()) { + throw tl::Exception (tl::to_string (QObject::tr ("Download failed: target directory '%1' does not exists").arg (dir.path ()))); + } + + for (WebDAVObject::iterator i = object.begin (); i != object.end (); ++i) { + + QFileInfo new_item (dir.absoluteFilePath (tl::to_qstring (i->name ()))); + + if (i->is_collection ()) { + + if (! new_item.exists ()) { + if (! dir.mkdir (tl::to_qstring (i->name ()))) { + throw tl::Exception (tl::to_string (QObject::tr ("Download failed: unable to create subdirectory '%2' in '%1'").arg (dir.path ()).arg (tl::to_qstring (i->name ())))); + } + } else if (! new_item.isDir ()) { + throw tl::Exception (tl::to_string (QObject::tr ("Download failed: unable to create subdirectory '%2' in '%1' - is already a file").arg (dir.path ()).arg (tl::to_qstring (i->name ())))); + } else if (! new_item.isWritable ()) { + throw tl::Exception (tl::to_string (QObject::tr ("Download failed: unable to create subdirectory '%2' in '%1' - no write permissions").arg (dir.path ()).arg (tl::to_qstring (i->name ())))); + } + + fetch_download_items (i->url (), tl::to_string (new_item.filePath ()), items, progress); + + } else { + + if (new_item.exists () && ! new_item.isWritable ()) { + throw tl::Exception (tl::to_string (QObject::tr ("Download failed: file is '%2' in '%1' - already exists, but no write permissions").arg (dir.path ()).arg (tl::to_qstring (i->name ())))); + } + + items.push_back (DownloadItem (i->url (), tl::to_string (dir.absoluteFilePath (tl::to_qstring (i->name ()))))); + + } + } + + } else { + items.push_back (DownloadItem (url, target)); + } +} + +bool +WebDAVObject::download (const std::string &url, const std::string &target) +{ + std::list items; + + try { + + tl::info << QObject::tr ("Fetching file structure from ") << url; + tl::AbsoluteProgress progress (tl::to_string (QObject::tr ("Fetching directory structure from %1").arg (tl::to_qstring (url)))); + fetch_download_items (url, target, items, progress); + + } catch (tl::Exception &ex) { + tl::error << QObject::tr ("Error downloading file structure from '") << url << "':" << tl::endl << ex.msg (); + return false; + } + + bool has_errors = false; + + { + tl::info << tl::to_string (QObject::tr ("Downloading %1 files now").arg (items.size ())); + tl::RelativeProgress progress (tl::to_string (QObject::tr ("Downloading file(s) from %1").arg (tl::to_qstring (url))), items.size (), 1); + for (std::list::const_iterator i = items.begin (); i != items.end (); ++i) { + + tl::info << QObject::tr ("Downloading '%1' to '%2' ..").arg (tl::to_qstring (i->url)).arg (tl::to_qstring (i->path)); + + try { + + tl::InputHttpStream http (i->url); + + QFile file (tl::to_qstring (i->path)); + if (! file.open (QIODevice::WriteOnly)) { + has_errors = true; + tl::error << QObject::tr ("Unable to open file '%1' for writing").arg (tl::to_qstring (i->path)); + } + + const size_t chunk = 65536; + char b[chunk]; + size_t read; + while ((read = http.read (b, sizeof (b))) > 0) { + if (! file.write (b, read)) { + tl::error << QObject::tr ("Unable to write %2 bytes file '%1'").arg (tl::to_qstring (i->path)).arg (int (read)); + has_errors = true; + break; + } + } + + file.close (); + + } catch (tl::Exception &ex) { + tl::error << QObject::tr ("Error downloading file from '") << i->url << "':" << tl::endl << ex.msg (); + has_errors = true; + } + + } + } + + return ! has_errors; +} + +} diff --git a/src/tl/tlWebDAV.h b/src/tl/tlWebDAV.h new file mode 100644 index 000000000..9e82ffb8e --- /dev/null +++ b/src/tl/tlWebDAV.h @@ -0,0 +1,152 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2017 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlWebDAV +#define HDR_tlWebDAV + +#include "tlCommon.h" +#include +#include + +namespace tl +{ + +/** + * @brief Represents an item in a WebDAV collection + */ +class TL_PUBLIC WebDAVItem +{ +public: + /** + * @brief Default constructor + */ + WebDAVItem () + : m_is_collection (false) + { + // .. nothing yet .. + } + + /** + * @brief Constructor + */ + WebDAVItem (bool is_collection, const std::string &url, const std::string &name) + : m_is_collection (is_collection), m_url (url), m_name (name) + { + // .. nothing yet .. + } + + /** + * @brief Gets a value indicating whether this item is a collection + * If false, it's a file. + */ + bool is_collection () const + { + return m_is_collection; + } + + /** + * @brief Gets the URL of this item + */ + const std::string &url () const + { + return m_url; + } + + /** + * @brief Gets the name of this item + * The name is only valid for sub-items. + */ + const std::string &name () const + { + return m_name; + } + +protected: + bool m_is_collection; + std::string m_url; + std::string m_name; +}; + +/** + * @brief Represents an object from a WebDAV URL + * This object can be a file or collection + */ +class TL_PUBLIC WebDAVObject + : public WebDAVItem +{ +public: + typedef std::vector container; + typedef container::const_iterator iterator; + + /** + * @brief Open a stream with the given URL + */ + WebDAVObject (); + + /** + * @brief Populates the collection from the given URL + * The depth value can be 0 (self only) or 1 (self + collection members). + */ + void read (const std::string &url, int depth); + + /** + * @brief Gets the items of this collection (begin iterator) + */ + iterator begin () const + { + return m_items.begin (); + } + + /** + * @brief Gets the items of this collection (begin iterator) + */ + iterator end () const + { + return m_items.end (); + } + + /** + * @brief Downloads the collection or file with the given URL + * + * This method will download the WebDAV object from url to the file path + * given in "target". + * + * For file download, the target must be the path of the target file. + * For collection download, the target must be a directory path. In this + * case, the target directory must exist already. + * + * Sub-directories are created if required. + * + * This method throws an exception if the directory structure could + * not be obtained or downloading of one file failed. + */ + static bool download (const std::string &url, const std::string &target); + +private: + container m_items; +}; + +} + +#endif + diff --git a/src/tl/tlXMLParser.h b/src/tl/tlXMLParser.h index 2760df1f0..6a1ae3015 100644 --- a/src/tl/tlXMLParser.h +++ b/src/tl/tlXMLParser.h @@ -671,12 +671,12 @@ public: return m_name; } - bool check_name (const std::string &, const std::string &, const std::string &qname) const + bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const { if (m_name == "*") { return true; } else { - return m_name == qname; // no namespace currently + return m_name == lname; // no namespace currently } } diff --git a/src/unit_tests/tlHttpStream.cc b/src/unit_tests/tlHttpStream.cc index 1783b2ccb..2c493de12 100644 --- a/src/unit_tests/tlHttpStream.cc +++ b/src/unit_tests/tlHttpStream.cc @@ -24,7 +24,8 @@ #include "tlHttpStream.h" #include "utHead.h" -std::string test_url1 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text"); +static std::string test_url1 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text"); +static std::string test_url2 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/dir1"); TEST(1) { @@ -36,3 +37,39 @@ TEST(1) EXPECT_EQ (res, "hello, world.\n"); } +TEST(2) +{ + tl::InputHttpStream stream (test_url2); + stream.add_header ("User-Agent", "SVN"); + stream.add_header ("Depth", "1"); + stream.set_request ("PROPFIND"); + stream.set_data (""); + + char b[10000]; + size_t n = stream.read (b, sizeof (b)); + std::string res (b, n); + + EXPECT_EQ (res, + "\n" + "\n" + "\n" + "/svn-public/klayout-resources/trunk/testdata/dir1/\n" + "\n" + "\n" + "\n" + "\n" + "HTTP/1.1 200 OK\n" + "\n" + "\n" + "\n" + "/svn-public/klayout-resources/trunk/testdata/dir1/text\n" + "\n" + "\n" + "\n" + "\n" + "HTTP/1.1 200 OK\n" + "\n" + "\n" + "\n" + ); +} diff --git a/src/unit_tests/tlWebDAV.cc b/src/unit_tests/tlWebDAV.cc new file mode 100644 index 000000000..bba9764f8 --- /dev/null +++ b/src/unit_tests/tlWebDAV.cc @@ -0,0 +1,129 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2017 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlWebDAV.h" +#include "utHead.h" + +#include + +static std::string test_url1 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata"); +static std::string test_url2 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text"); + +static std::string collection2string (const tl::WebDAVObject &coll) +{ + std::string s; + for (tl::WebDAVObject::iterator c = coll.begin (); c != coll.end (); ++c) { + if (!s.empty ()) { + s += "\n"; + } + if (c->is_collection ()) { + s += "[dir] "; + } + s += c->name (); + s += " "; + s += c->url (); + } + return s; +} + +TEST(1) +{ + tl::WebDAVObject collection; + collection.read (test_url1, 1); + + EXPECT_EQ (collection.is_collection (), true); + EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/"); + + EXPECT_EQ (collection2string (collection), + "[dir] dir1 http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/dir1/\n" + "[dir] dir2 http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/dir2/\n" + "text http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text\n" + "text2 http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text2" + ); +} + +TEST(2) +{ + tl::WebDAVObject collection; + collection.read (test_url1, 0); + + EXPECT_EQ (collection.is_collection (), true); + EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/"); + EXPECT_EQ (collection2string (collection), ""); +} + +TEST(3) +{ + tl::WebDAVObject collection; + collection.read (test_url2, 1); + + EXPECT_EQ (collection.is_collection (), false); + EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text"); + EXPECT_EQ (collection2string (collection), ""); +} + +TEST(4) +{ + tl::WebDAVObject collection; + collection.read (test_url2, 0); + + EXPECT_EQ (collection.is_collection (), false); + EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text"); + EXPECT_EQ (collection2string (collection), ""); +} + +TEST(5) +{ + tl::WebDAVObject collection; + + QDir tmp_dir (tl::to_qstring (tmp_file ("tmp"))); + EXPECT_EQ (tmp_dir.exists (), false); + + tmp_dir.cdUp (); + tmp_dir.mkdir (tl::to_qstring ("tmp")); + tmp_dir.cd (tl::to_qstring ("tmp")); + + bool res = collection.download (test_url1, tl::to_string (tmp_dir.absolutePath ())); + EXPECT_EQ (res, true); + + QDir dir1 (tmp_dir.absoluteFilePath (QString::fromUtf8 ("dir1"))); + QDir dir2 (tmp_dir.absoluteFilePath (QString::fromUtf8 ("dir2"))); + QDir dir21 (dir2.absoluteFilePath (QString::fromUtf8 ("dir21"))); + EXPECT_EQ (dir1.exists (), true); + EXPECT_EQ (dir2.exists (), true); + EXPECT_EQ (dir21.exists (), true); + + QByteArray ba; + + QFile text1 (dir1.absoluteFilePath (QString::fromUtf8 ("text"))); + text1.open (QIODevice::ReadOnly); + ba = text1.read (10000); + EXPECT_EQ (ba.constData (), "A text.\n"); + text1.close (); + + QFile text21 (dir21.absoluteFilePath (QString::fromUtf8 ("text"))); + text21.open (QIODevice::ReadOnly); + ba = text21.read (10000); + EXPECT_EQ (ba.constData (), "A text II.I.\n"); + text21.close (); +} diff --git a/src/unit_tests/unit_tests.pro b/src/unit_tests/unit_tests.pro index 608f86786..85b821dd9 100644 --- a/src/unit_tests/unit_tests.pro +++ b/src/unit_tests/unit_tests.pro @@ -98,7 +98,8 @@ SOURCES = \ tlFileSystemWatcher.cc \ laySalt.cc \ tlFileUtils.cc \ - tlHttpStream.cc + tlHttpStream.cc \ + tlWebDAV.cc # main components: SOURCES += \