WIP: downloading of packages

- Support for WebDAV
- Download manager implemented
- Apply button functionality implemented

(needs testing)
This commit is contained in:
Matthias Koefferlein 2017-03-27 15:46:01 +02:00
parent d98495c18a
commit cb589dc2d3
17 changed files with 1130 additions and 32 deletions

View File

@ -0,0 +1,156 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>SaltManagerInstallConfirmationDialog</class>
<widget class="QDialog" name="SaltManagerInstallConfirmationDialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>495</width>
<height>478</height>
</rect>
</property>
<property name="windowTitle">
<string>Ready for Installation</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QLabel" name="label">
<property name="text">
<string>The following packages are now ready for installation or update:</string>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<widget class="QTreeWidget" name="list">
<column>
<property name="text">
<string>Package</string>
</property>
</column>
<column>
<property name="text">
<string>Action</string>
</property>
</column>
<column>
<property name="text">
<string>Version</string>
</property>
</column>
<column>
<property name="text">
<string>Download link</string>
</property>
</column>
</widget>
</item>
<item>
<widget class="QLabel" name="label_2">
<property name="text">
<string>Press &quot;Ok&quot; to install or update these packages or &quot;Cancel&quot; to abort.</string>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeType">
<enum>QSizePolicy::Fixed</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>6</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QDialogButtonBox" name="buttonBox">
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
<action name="actionNew">
<property name="icon">
<iconset resource="layResources.qrc">
<normaloff>:/add.png</normaloff>:/add.png</iconset>
</property>
<property name="text">
<string>New</string>
</property>
<property name="toolTip">
<string>New package</string>
</property>
</action>
<action name="actionDelete">
<property name="icon">
<iconset resource="layResources.qrc">
<normaloff>:/clear.png</normaloff>:/clear.png</iconset>
</property>
<property name="text">
<string>Delete</string>
</property>
<property name="toolTip">
<string>Delete package</string>
</property>
</action>
<action name="actionImport">
<property name="icon">
<iconset resource="layResources.qrc">
<normaloff>:/import.png</normaloff>:/import.png</iconset>
</property>
<property name="text">
<string>Import</string>
</property>
<property name="toolTip">
<string>Import package</string>
</property>
</action>
</widget>
<resources>
<include location="layResources.qrc"/>
</resources>
<connections>
<connection>
<sender>buttonBox</sender>
<signal>accepted()</signal>
<receiver>SaltManagerInstallConfirmationDialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>273</x>
<y>431</y>
</hint>
<hint type="destinationlabel">
<x>276</x>
<y>448</y>
</hint>
</hints>
</connection>
<connection>
<sender>buttonBox</sender>
<signal>rejected()</signal>
<receiver>SaltManagerInstallConfirmationDialog</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>351</x>
<y>426</y>
</hint>
<hint type="destinationlabel">
<x>363</x>
<y>445</y>
</hint>
</hints>
</connection>
</connections>
</ui>

View File

@ -101,7 +101,8 @@ FORMS = \
MainConfigPage7.ui \
SaltManagerDialog.ui \
SaltGrainPropertiesDialog.ui \
SaltGrainTemplateSelectionDialog.ui
SaltGrainTemplateSelectionDialog.ui \
SaltManagerInstallConfirmationDialog.ui
SOURCES = \
gsiDeclLayApplication.cc \

View File

@ -21,11 +21,11 @@
*/
#include "laySalt.h"
#include "laySaltDownloadManager.h"
#include "tlString.h"
#include "tlFileUtils.h"
#include "tlLog.h"
#include "tlInternational.h"
#include "tlWebDAV.h"
#include <QFileInfo>
#include <QDir>
@ -307,12 +307,23 @@ public:
}
bool
Salt::create_grain (const SaltGrain &templ, SaltGrain &target, SaltDownloadManager *download_manager)
Salt::create_grain (const SaltGrain &templ, SaltGrain &target)
{
tl_assert (!m_root.is_empty ());
const SaltGrains *coll = m_root.begin_collections ().operator-> ();
if (target.name ().empty ()) {
target.set_name (templ.name ());
}
if (target.path ().empty ()) {
lay::SaltGrain *g = grain_by_name (target.name ());
if (g) {
target.set_path (g->path ());
}
}
std::string path = target.path ();
if (! path.empty ()) {
coll = 0;
@ -383,11 +394,11 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, SaltDownloadManag
} else if (! templ.url ().empty ()) {
tl_assert (download_manager != 0);
// otherwise download from the URL
tl::info << QObject::tr ("Downloading package from '%1' to '%2' ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ()));
res = download_manager->download (templ.url (), target.path ());
res = tl::WebDAVObject::download (templ.url (), target.path ());
target.set_url (templ.url ());
}

View File

@ -34,8 +34,6 @@
namespace lay
{
class SaltDownloadManager;
/**
* @brief The global salt (package manager) object
* This object can be configured to represent a couple of locations.
@ -123,6 +121,14 @@ public:
*/
SaltGrain *grain_by_name (const std::string &name);
/**
* @brief Gets the grain with the given name (const version)
*/
const SaltGrain *grain_by_name (const std::string &name) const
{
return const_cast<Salt *> (this)->grain_by_name (name);
}
/**
* @brief Loads the salt from a "salt mine" file
*/
@ -165,11 +171,11 @@ public:
* all files related to this grain. It will copy the download URL from the template into the
* new grain, so updates will come from the original location.
*
* The target's name must be set. If a specific target location is desired, the target's
* path must be set too.
*
* This method refuses to overwrite existing grains, so an update needs to be performed by first
* deleting the grain and then re-installing it.
* If the target's name is not set, it will be taken from the template.
* If the target's path is not set and a grain with the given name already exists in
* the package, the path is taken from that grain.
* If no target path is set and no grain with this name exists yet, a new path will
* be constructed using the first location in the salt.
*
* The target grain will be updated with the installation information. If the target grain
* contains an installation path prior to the installation, this path will be used for the
@ -177,7 +183,7 @@ public:
*
* Returns true, if the package could be created successfully.
*/
bool create_grain (const SaltGrain &templ, SaltGrain &target, SaltDownloadManager *download_manager = 0);
bool create_grain (const SaltGrain &templ, SaltGrain &target);
signals:
/**

View File

@ -21,20 +21,189 @@
*/
#include "laySaltDownloadManager.h"
#include "laySalt.h"
#include "tlFileUtils.h"
#include "tlWebDAV.h"
#include "ui_SaltManagerInstallConfirmationDialog.h"
#include <QTreeWidgetItem>
namespace lay
{
// ----------------------------------------------------------------------------------
class ConfirmationDialog
: public QDialog, private Ui::SaltManagerInstallConfirmationDialog
{
public:
ConfirmationDialog (QWidget *parent)
: QDialog (parent)
{
Ui::SaltManagerInstallConfirmationDialog::setupUi (this);
}
void add_info (const std::string &name, bool update, const std::string &version, const std::string &url)
{
QTreeWidgetItem *item = new QTreeWidgetItem (list);
item->setText (0, tl::to_qstring (name));
item->setText (1, update ? tr ("UPDATE") : tr ("INSTALL"));
item->setText (2, tl::to_qstring (version));
item->setText (3, tl::to_qstring (url));
}
};
// ----------------------------------------------------------------------------------
SaltDownloadManager::SaltDownloadManager ()
{
// .. nothing yet ..
}
bool
SaltDownloadManager::download (const std::string &url, const std::string &target_dir)
void
SaltDownloadManager::register_download (const std::string &name, const std::string &url, const std::string &version)
{
// @@@
m_registry.insert (std::make_pair (name, Descriptor (url, version)));
}
void
SaltDownloadManager::compute_dependencies (const lay::Salt &salt, const lay::Salt &salt_mine)
{
tl::AbsoluteProgress progress (tl::to_string (QObject::tr ("Computing package dependencies ..")));
while (needs_iteration ()) {
fetch_missing (salt_mine, progress);
std::map<std::string, Descriptor> registry = m_registry;
for (std::map<std::string, Descriptor>::const_iterator p = registry.begin (); p != registry.end (); ++p) {
for (std::vector<SaltGrain::Dependency>::const_iterator d = p->second.grain.dependencies ().begin (); d != p->second.grain.dependencies ().end (); ++d) {
std::map<std::string, Descriptor>::iterator r = m_registry.find (d->name);
if (r != m_registry.end ()) {
if (SaltGrain::compare_versions (r->second.version, d->version) < 0) {
// Grain is present, but too old -> update version and reload in the next iteration
r->second.downloaded = false;
r->second.version = d->version;
r->second.url = d->url;
r->second.downloaded = false;
}
} else {
const SaltGrain *g = salt.grain_by_name (d->name);
if (g) {
// Grain is installed already, but too old -> register for update
if (SaltGrain::compare_versions (g->version (), d->version) < 0) {
register_download (d->name, d->url, d->version);
}
} else {
register_download (d->name, d->url, d->version);
}
}
}
}
}
}
bool
SaltDownloadManager::needs_iteration ()
{
for (std::map<std::string, Descriptor>::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) {
if (! p->second.downloaded) {
return true;
}
}
return false;
}
void
SaltDownloadManager::fetch_missing (const lay::Salt &salt_mine, tl::AbsoluteProgress &progress)
{
for (std::map<std::string, Descriptor>::iterator p = m_registry.begin (); p != m_registry.end (); ++p) {
if (! p->second.downloaded) {
++progress;
// If no URL is given, utilize the salt mine to fetch it
if (p->second.url.empty ()) {
const lay::SaltGrain *g = salt_mine.grain_by_name (p->first);
if (SaltGrain::compare_versions (g->version (), p->second.version) < 0) {
throw tl::Exception (tl::to_string (QObject::tr ("Package '%1': package in repository is too old (%2) to satisfy requirements (%3)").arg (tl::to_qstring (p->first)).arg (tl::to_qstring (g->version ())).arg (tl::to_qstring (p->second.version))));
}
p->second.version = g->version ();
p->second.url = g->url ();
}
p->second.grain = SaltGrain::from_url (p->second.url);
p->second.downloaded = true;
}
}
}
bool
SaltDownloadManager::show_confirmation_dialog (QWidget *parent, const lay::Salt &salt)
{
lay::ConfirmationDialog dialog (parent);
// First the packages to update
for (std::map<std::string, Descriptor>::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) {
const lay::SaltGrain *g = salt.grain_by_name (p->first);
if (g) {
dialog.add_info (p->first, true, g->version () + "->" + p->second.version, p->second.url);
}
}
// Then the packages to install
for (std::map<std::string, Descriptor>::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) {
const lay::SaltGrain *g = salt.grain_by_name (p->first);
if (!g) {
dialog.add_info (p->first, false, p->second.version, p->second.url);
}
}
return dialog.exec ();
}
bool
SaltDownloadManager::execute (lay::Salt &salt)
{
bool result = true;
tl::RelativeProgress progress (tl::to_string (QObject::tr ("Downloading packages")), m_registry.size (), 1);
for (std::map<std::string, Descriptor>::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) {
lay::SaltGrain target;
target.set_name (p->first);
lay::SaltGrain *g = salt.grain_by_name (p->first);
if (g) {
target.set_path (g->path ());
}
if (! salt.create_grain (p->second.grain, target)) {
result = false;
}
++progress;
}
return result;
}
}

View File

@ -24,17 +24,27 @@
#define HDR_laySaltDownloadManager
#include "layCommon.h"
#include "laySaltGrain.h"
#include "tlProgress.h"
#include <QObject>
#include <string>
#include <map>
namespace lay
{
class Salt;
/**
* @brief The download manager
*
* This class is responsible for handling the downloads for
* grains.
* grains. The basic sequence is:
* + "register_download" (multiple times) to register the packages intended for download
* + "compute_dependencies" to determine all related packages
* + (optional) "show_confirmation_dialog"
* + "execute" to actually execute the downloads
*/
class LAY_PUBLIC SaltDownloadManager
: public QObject
@ -48,11 +58,58 @@ public:
SaltDownloadManager ();
/**
* @brief Downloads the files from the given URL to the given target location
* The target directory needs to exist.
* Returns true, if the download was successful, false otherwise.
* @brief Registers an URL (with version) for download in the given target directory
*
* The target directory can be empty. In this case, the downloader will pick an approriate one.
*/
bool download (const std::string &url, const std::string &target_dir);
void register_download (const std::string &name, const std::string &url, const std::string &version);
/**
* @brief Computes the dependencies after all required packages have been registered
*
* This method will compute the dependencies. Packages not present in the list of
* packages ("salt" argument), will be scheduled for download too. Dependency packages
* are looked up in "salt_mine" if no download URL is given.
*/
void compute_dependencies (const lay::Salt &salt, const Salt &salt_mine);
/**
* @brief Presents a dialog showing the packages scheduled for download
*
* This method requires all dependencies to be computed. It will return false
* if the dialog is not confirmed.
*
* "salt" needs to be the currently installed packages so the dialog can
* indicate which packages will be updated.
*/
bool show_confirmation_dialog (QWidget *parent, const lay::Salt &salt);
/**
* @brief Actually execute the downloads
*
* This method will return false if anything goes wrong.
* Failed packages will be removed entirely after they have been listed in
* an error dialog.
*/
bool execute (lay::Salt &salt);
private:
struct Descriptor
{
Descriptor (const std::string &_url, const std::string &_version)
: url (_url), version (_version), downloaded (false)
{ }
std::string url;
std::string version;
bool downloaded;
lay::SaltGrain grain;
};
std::map<std::string, Descriptor> m_registry;
bool needs_iteration ();
void fetch_missing (const lay::Salt &salt_mine, tl::AbsoluteProgress &progress);
};
}

View File

@ -23,6 +23,7 @@
#include "laySaltManagerDialog.h"
#include "laySaltModel.h"
#include "laySaltGrainPropertiesDialog.h"
#include "laySaltDownloadManager.h"
#include "laySalt.h"
#include "ui_SaltGrainTemplateSelectionDialog.h"
#include "tlString.h"
@ -151,6 +152,7 @@ SaltManagerDialog::SaltManagerDialog (QWidget *parent)
connect (edit_button, SIGNAL (clicked ()), this, SLOT (edit_properties ()));
connect (create_button, SIGNAL (clicked ()), this, SLOT (create_grain ()));
connect (delete_button, SIGNAL (clicked ()), this, SLOT (delete_grain ()));
connect (apply_button, SIGNAL (clicked ()), this, SLOT (apply ()));
mp_salt = get_salt ();
mp_salt_mine = get_salt_mine ();
@ -258,6 +260,42 @@ SaltManagerDialog::mark_clicked ()
model->set_marked (g->name (), !model->is_marked (g->name ()));
}
void
SaltManagerDialog::apply ()
{
BEGIN_PROTECTED
lay::SaltDownloadManager manager;
bool any = false;
// fetch all marked grains and register for download
SaltModel *model = dynamic_cast <SaltModel *> (salt_mine_view->model ());
if (model) {
for (int i = model->rowCount (QModelIndex ()); i > 0; ) {
--i;
QModelIndex index = model->index (i, 0, QModelIndex ());
SaltGrain *g = model->grain_from_index (index);
if (g && model->is_marked (g->name ())) {
manager.register_download (g->name (), g->url (), g->version ());
any = true;
}
}
}
if (! any) {
throw tl::Exception (tl::to_string (tr ("No packages marked for installation or update")));
}
manager.compute_dependencies (*mp_salt, *mp_salt_mine);
if (manager.show_confirmation_dialog (this, *mp_salt)) {
manager.execute (*mp_salt);
}
END_PROTECTED
}
void
SaltManagerDialog::edit_properties ()
{

View File

@ -95,6 +95,11 @@ private slots:
*/
void mode_changed ();
/**
* @brief Called when the "apply" button is clicked
*/
void apply ();
/**
* @brief Called when one search text changed
*/

View File

@ -40,7 +40,8 @@ SOURCES = \
tlXMLParser.cc \
tlXMLWriter.cc \
tlFileSystemWatcher.cc \
tlFileUtils.cc
tlFileUtils.cc \
tlWebDAV.cc
HEADERS = \
tlAlgorithm.h \
@ -83,7 +84,8 @@ HEADERS = \
tlXMLWriter.h \
tlFileSystemWatcher.h \
tlCommon.h \
tlFileUtils.h
tlFileUtils.h \
tlWebDAV.h
INCLUDEPATH =
DEPENDPATH =

View File

@ -87,7 +87,6 @@ InputHttpStream::InputHttpStream (const std::string &url)
connect (s_network_manager, SIGNAL (finished (QNetworkReply *)), this, SLOT (finished (QNetworkReply *)));
connect (s_network_manager, SIGNAL (authenticationRequired (QNetworkReply *, QAuthenticator *)), this, SLOT (authenticationRequired (QNetworkReply *, QAuthenticator *)));
connect (s_network_manager, SIGNAL (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)), this, SLOT (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)));
issue_request (QUrl (tl::to_qstring (url)));
mp_reply = 0;
}
@ -115,6 +114,12 @@ InputHttpStream::set_data (const char *data, size_t n)
m_data = QByteArray (data, int (n));
}
void
InputHttpStream::add_header (const std::string &name, const std::string &value)
{
m_headers.insert (std::make_pair (name, value));
}
void
InputHttpStream::authenticationRequired (QNetworkReply *reply, QAuthenticator *auth)
{
@ -148,19 +153,26 @@ InputHttpStream::issue_request (const QUrl &url)
delete mp_buffer;
mp_buffer = 0;
QNetworkRequest request (url);
for (std::map<std::string, std::string>::const_iterator h = m_headers.begin (); h != m_headers.end (); ++h) {
request.setRawHeader (QByteArray (h->first.c_str ()), QByteArray (h->second.c_str ()));
}
if (m_data.isEmpty ()) {
s_network_manager->sendCustomRequest (QNetworkRequest (url), m_request);
s_network_manager->sendCustomRequest (request, m_request);
} else {
mp_buffer = new QBuffer (&m_data);
s_network_manager->sendCustomRequest (QNetworkRequest (url), m_request, mp_buffer);
s_network_manager->sendCustomRequest (request, m_request, mp_buffer);
}
}
size_t
InputHttpStream::read (char *b, size_t n)
{
if (mp_reply == 0) {
issue_request (QUrl (tl::to_qstring (m_url)));
}
while (mp_reply == 0) {
QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents);
QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents, 100);
}
if (mp_reply->error () != QNetworkReply::NoError) {

View File

@ -90,6 +90,11 @@ public:
*/
void set_data (const char *data, size_t n);
/**
* @brief Sets a header field
*/
void add_header (const std::string &name, const std::string &value);
/**
* @brief Read from the stream
* Implements the basic read method.
@ -121,6 +126,7 @@ private:
QByteArray m_request;
QByteArray m_data;
QBuffer *mp_buffer;
std::map<std::string, std::string> m_headers;
void issue_request (const QUrl &url);
};

316
src/tl/tlWebDAV.cc Normal file
View File

@ -0,0 +1,316 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2017 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "tlWebDAV.h"
#include "tlXMLParser.h"
#include "tlHttpStream.h"
#include "tlStream.h"
#include "tlInternational.h"
#include "tlProgress.h"
#include "tlLog.h"
#include <QUrl>
#include <QDir>
namespace tl
{
// ---------------------------------------------------------------
// WebDAVCollection implementation
WebDAVObject::WebDAVObject ()
{
// .. nothing yet ..
}
namespace
{
/**
* @brief A dummy "DOM" for the WebDAV reply
*/
struct ResourceType
{
ResourceType () : is_collection (false) { }
const std::string &collection () const
{
static std::string empty;
return empty;
}
void set_collection (const std::string &)
{
is_collection = true;
}
bool is_collection;
};
/**
* @brief A dummy "DOM" for the WebDAV reply
*/
struct Prop
{
ResourceType resourcetype;
};
/**
* @brief A dummy "DOM" for the WebDAV reply
*/
struct PropStat
{
std::string status;
Prop prop;
};
/**
* @brief A dummy "DOM" for the WebDAV reply
*/
struct Response
{
std::string href;
PropStat propstat;
};
/**
* @brief A dummy "DOM" for the WebDAV reply
*/
struct MultiStatus
{
typedef std::list<Response> container;
typedef container::const_iterator iterator;
iterator begin () const { return responses.begin (); }
iterator end () const { return responses.end (); }
void add (const Response &r) { responses.push_back (r); }
container responses;
};
}
tl::XMLStruct<MultiStatus> xml_struct ("multistatus",
tl::make_element (&MultiStatus::begin, &MultiStatus::end, &MultiStatus::add, "response",
tl::make_member (&Response::href, "href") +
tl::make_element (&Response::propstat, "propstat",
tl::make_member (&PropStat::status, "status") +
tl::make_element (&PropStat::prop, "prop",
tl::make_element (&Prop::resourcetype, "resourcetype",
tl::make_member (&ResourceType::collection, &ResourceType::set_collection, "collection")
)
)
)
)
);
static std::string item_name (const QString &path1, const QString &path2)
{
QStringList sl1 = path1.split (QChar ('/'));
if (! sl1.empty () && sl1.back ().isEmpty ()) {
sl1.pop_back ();
}
QStringList sl2 = path2.split (QChar ('/'));
if (! sl2.empty () && sl2.back ().isEmpty ()) {
sl2.pop_back ();
}
int i = 0;
for ( ; i < sl1.length () && i < sl2.length (); ++i) {
if (sl1 [i] != sl2 [i]) {
throw tl::Exception (tl::to_string (QObject::tr ("Invalid WebDAV response: %1 is not a collection item of %2").arg (path2).arg (path1)));
}
}
if (i == sl2.length ()) {
return std::string ();
} else if (i + 1 == sl2.length ()) {
return tl::to_string (sl2[i]);
} else {
throw tl::Exception (tl::to_string (QObject::tr ("Invalid WebDAV response: %1 is not a collection sub-item of %2").arg (path2).arg (path1)));
}
}
void
WebDAVObject::read (const std::string &url, int depth)
{
QUrl base_url = QUrl (tl::to_qstring (url));
tl::InputHttpStream http (url);
http.add_header ("User-Agent", "SVN");
http.add_header ("Depth", tl::to_string (depth));
http.set_request ("PROPFIND");
http.set_data ("<?xml version=\"1.0\" encoding=\"utf-8\"?><propfind xmlns=\"DAV:\"><prop><resourcetype xmlns=\"DAV:\"/></prop></propfind>");
MultiStatus multistatus;
tl::InputStream stream (http);
tl::XMLStreamSource source (stream);
xml_struct.parse (source, multistatus);
// TODO: check status ..
m_items.clear ();
for (MultiStatus::iterator r = multistatus.begin (); r != multistatus.end (); ++r) {
bool is_collection = r->propstat.prop.resourcetype.is_collection;
QUrl item_url = base_url.resolved (QUrl (tl::to_qstring (r->href)));
std::string n = item_name (base_url.path (), item_url.path ());
std::string item_url_string = tl::to_string (item_url.toString ());
if (! n.empty ()) {
m_items.push_back (WebDAVItem (is_collection, item_url_string, n));
} else {
m_is_collection = is_collection;
m_url = item_url_string;
}
}
}
namespace
{
struct DownloadItem
{
DownloadItem (const std::string &u, const std::string &p)
{
url = u;
path = p;
}
std::string url;
std::string path;
};
}
static
void fetch_download_items (const std::string &url, const std::string &target, std::list<DownloadItem> &items, tl::AbsoluteProgress &progress)
{
++progress;
WebDAVObject object;
object.read (url, 1);
if (object.is_collection ()) {
QDir dir (tl::to_qstring (target));
if (! dir.exists ()) {
throw tl::Exception (tl::to_string (QObject::tr ("Download failed: target directory '%1' does not exists").arg (dir.path ())));
}
for (WebDAVObject::iterator i = object.begin (); i != object.end (); ++i) {
QFileInfo new_item (dir.absoluteFilePath (tl::to_qstring (i->name ())));
if (i->is_collection ()) {
if (! new_item.exists ()) {
if (! dir.mkdir (tl::to_qstring (i->name ()))) {
throw tl::Exception (tl::to_string (QObject::tr ("Download failed: unable to create subdirectory '%2' in '%1'").arg (dir.path ()).arg (tl::to_qstring (i->name ()))));
}
} else if (! new_item.isDir ()) {
throw tl::Exception (tl::to_string (QObject::tr ("Download failed: unable to create subdirectory '%2' in '%1' - is already a file").arg (dir.path ()).arg (tl::to_qstring (i->name ()))));
} else if (! new_item.isWritable ()) {
throw tl::Exception (tl::to_string (QObject::tr ("Download failed: unable to create subdirectory '%2' in '%1' - no write permissions").arg (dir.path ()).arg (tl::to_qstring (i->name ()))));
}
fetch_download_items (i->url (), tl::to_string (new_item.filePath ()), items, progress);
} else {
if (new_item.exists () && ! new_item.isWritable ()) {
throw tl::Exception (tl::to_string (QObject::tr ("Download failed: file is '%2' in '%1' - already exists, but no write permissions").arg (dir.path ()).arg (tl::to_qstring (i->name ()))));
}
items.push_back (DownloadItem (i->url (), tl::to_string (dir.absoluteFilePath (tl::to_qstring (i->name ())))));
}
}
} else {
items.push_back (DownloadItem (url, target));
}
}
bool
WebDAVObject::download (const std::string &url, const std::string &target)
{
std::list<DownloadItem> items;
try {
tl::info << QObject::tr ("Fetching file structure from ") << url;
tl::AbsoluteProgress progress (tl::to_string (QObject::tr ("Fetching directory structure from %1").arg (tl::to_qstring (url))));
fetch_download_items (url, target, items, progress);
} catch (tl::Exception &ex) {
tl::error << QObject::tr ("Error downloading file structure from '") << url << "':" << tl::endl << ex.msg ();
return false;
}
bool has_errors = false;
{
tl::info << tl::to_string (QObject::tr ("Downloading %1 files now").arg (items.size ()));
tl::RelativeProgress progress (tl::to_string (QObject::tr ("Downloading file(s) from %1").arg (tl::to_qstring (url))), items.size (), 1);
for (std::list<DownloadItem>::const_iterator i = items.begin (); i != items.end (); ++i) {
tl::info << QObject::tr ("Downloading '%1' to '%2' ..").arg (tl::to_qstring (i->url)).arg (tl::to_qstring (i->path));
try {
tl::InputHttpStream http (i->url);
QFile file (tl::to_qstring (i->path));
if (! file.open (QIODevice::WriteOnly)) {
has_errors = true;
tl::error << QObject::tr ("Unable to open file '%1' for writing").arg (tl::to_qstring (i->path));
}
const size_t chunk = 65536;
char b[chunk];
size_t read;
while ((read = http.read (b, sizeof (b))) > 0) {
if (! file.write (b, read)) {
tl::error << QObject::tr ("Unable to write %2 bytes file '%1'").arg (tl::to_qstring (i->path)).arg (int (read));
has_errors = true;
break;
}
}
file.close ();
} catch (tl::Exception &ex) {
tl::error << QObject::tr ("Error downloading file from '") << i->url << "':" << tl::endl << ex.msg ();
has_errors = true;
}
}
}
return ! has_errors;
}
}

152
src/tl/tlWebDAV.h Normal file
View File

@ -0,0 +1,152 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2017 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef HDR_tlWebDAV
#define HDR_tlWebDAV
#include "tlCommon.h"
#include <string>
#include <vector>
namespace tl
{
/**
* @brief Represents an item in a WebDAV collection
*/
class TL_PUBLIC WebDAVItem
{
public:
/**
* @brief Default constructor
*/
WebDAVItem ()
: m_is_collection (false)
{
// .. nothing yet ..
}
/**
* @brief Constructor
*/
WebDAVItem (bool is_collection, const std::string &url, const std::string &name)
: m_is_collection (is_collection), m_url (url), m_name (name)
{
// .. nothing yet ..
}
/**
* @brief Gets a value indicating whether this item is a collection
* If false, it's a file.
*/
bool is_collection () const
{
return m_is_collection;
}
/**
* @brief Gets the URL of this item
*/
const std::string &url () const
{
return m_url;
}
/**
* @brief Gets the name of this item
* The name is only valid for sub-items.
*/
const std::string &name () const
{
return m_name;
}
protected:
bool m_is_collection;
std::string m_url;
std::string m_name;
};
/**
* @brief Represents an object from a WebDAV URL
* This object can be a file or collection
*/
class TL_PUBLIC WebDAVObject
: public WebDAVItem
{
public:
typedef std::vector<WebDAVItem> container;
typedef container::const_iterator iterator;
/**
* @brief Open a stream with the given URL
*/
WebDAVObject ();
/**
* @brief Populates the collection from the given URL
* The depth value can be 0 (self only) or 1 (self + collection members).
*/
void read (const std::string &url, int depth);
/**
* @brief Gets the items of this collection (begin iterator)
*/
iterator begin () const
{
return m_items.begin ();
}
/**
* @brief Gets the items of this collection (begin iterator)
*/
iterator end () const
{
return m_items.end ();
}
/**
* @brief Downloads the collection or file with the given URL
*
* This method will download the WebDAV object from url to the file path
* given in "target".
*
* For file download, the target must be the path of the target file.
* For collection download, the target must be a directory path. In this
* case, the target directory must exist already.
*
* Sub-directories are created if required.
*
* This method throws an exception if the directory structure could
* not be obtained or downloading of one file failed.
*/
static bool download (const std::string &url, const std::string &target);
private:
container m_items;
};
}
#endif

View File

@ -671,12 +671,12 @@ public:
return m_name;
}
bool check_name (const std::string &, const std::string &, const std::string &qname) const
bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const
{
if (m_name == "*") {
return true;
} else {
return m_name == qname; // no namespace currently
return m_name == lname; // no namespace currently
}
}

View File

@ -24,7 +24,8 @@
#include "tlHttpStream.h"
#include "utHead.h"
std::string test_url1 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text");
static std::string test_url1 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text");
static std::string test_url2 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/dir1");
TEST(1)
{
@ -36,3 +37,39 @@ TEST(1)
EXPECT_EQ (res, "hello, world.\n");
}
TEST(2)
{
tl::InputHttpStream stream (test_url2);
stream.add_header ("User-Agent", "SVN");
stream.add_header ("Depth", "1");
stream.set_request ("PROPFIND");
stream.set_data ("<?xml version=\"1.0\" encoding=\"utf-8\"?><propfind xmlns=\"DAV:\"><prop><resourcetype xmlns=\"DAV:\"/></prop></propfind>");
char b[10000];
size_t n = stream.read (b, sizeof (b));
std::string res (b, n);
EXPECT_EQ (res,
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
"<D:multistatus xmlns:D=\"DAV:\" xmlns:ns0=\"DAV:\">\n"
"<D:response xmlns:lp1=\"DAV:\">\n"
"<D:href>/svn-public/klayout-resources/trunk/testdata/dir1/</D:href>\n"
"<D:propstat>\n"
"<D:prop>\n"
"<lp1:resourcetype><D:collection/></lp1:resourcetype>\n"
"</D:prop>\n"
"<D:status>HTTP/1.1 200 OK</D:status>\n"
"</D:propstat>\n"
"</D:response>\n"
"<D:response xmlns:lp1=\"DAV:\">\n"
"<D:href>/svn-public/klayout-resources/trunk/testdata/dir1/text</D:href>\n"
"<D:propstat>\n"
"<D:prop>\n"
"<lp1:resourcetype/>\n"
"</D:prop>\n"
"<D:status>HTTP/1.1 200 OK</D:status>\n"
"</D:propstat>\n"
"</D:response>\n"
"</D:multistatus>\n"
);
}

129
src/unit_tests/tlWebDAV.cc Normal file
View File

@ -0,0 +1,129 @@
/*
KLayout Layout Viewer
Copyright (C) 2006-2017 Matthias Koefferlein
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "tlWebDAV.h"
#include "utHead.h"
#include <QDir>
static std::string test_url1 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata");
static std::string test_url2 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text");
static std::string collection2string (const tl::WebDAVObject &coll)
{
std::string s;
for (tl::WebDAVObject::iterator c = coll.begin (); c != coll.end (); ++c) {
if (!s.empty ()) {
s += "\n";
}
if (c->is_collection ()) {
s += "[dir] ";
}
s += c->name ();
s += " ";
s += c->url ();
}
return s;
}
TEST(1)
{
tl::WebDAVObject collection;
collection.read (test_url1, 1);
EXPECT_EQ (collection.is_collection (), true);
EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/");
EXPECT_EQ (collection2string (collection),
"[dir] dir1 http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/dir1/\n"
"[dir] dir2 http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/dir2/\n"
"text http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text\n"
"text2 http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text2"
);
}
TEST(2)
{
tl::WebDAVObject collection;
collection.read (test_url1, 0);
EXPECT_EQ (collection.is_collection (), true);
EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/");
EXPECT_EQ (collection2string (collection), "");
}
TEST(3)
{
tl::WebDAVObject collection;
collection.read (test_url2, 1);
EXPECT_EQ (collection.is_collection (), false);
EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text");
EXPECT_EQ (collection2string (collection), "");
}
TEST(4)
{
tl::WebDAVObject collection;
collection.read (test_url2, 0);
EXPECT_EQ (collection.is_collection (), false);
EXPECT_EQ (collection.url (), "http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text");
EXPECT_EQ (collection2string (collection), "");
}
TEST(5)
{
tl::WebDAVObject collection;
QDir tmp_dir (tl::to_qstring (tmp_file ("tmp")));
EXPECT_EQ (tmp_dir.exists (), false);
tmp_dir.cdUp ();
tmp_dir.mkdir (tl::to_qstring ("tmp"));
tmp_dir.cd (tl::to_qstring ("tmp"));
bool res = collection.download (test_url1, tl::to_string (tmp_dir.absolutePath ()));
EXPECT_EQ (res, true);
QDir dir1 (tmp_dir.absoluteFilePath (QString::fromUtf8 ("dir1")));
QDir dir2 (tmp_dir.absoluteFilePath (QString::fromUtf8 ("dir2")));
QDir dir21 (dir2.absoluteFilePath (QString::fromUtf8 ("dir21")));
EXPECT_EQ (dir1.exists (), true);
EXPECT_EQ (dir2.exists (), true);
EXPECT_EQ (dir21.exists (), true);
QByteArray ba;
QFile text1 (dir1.absoluteFilePath (QString::fromUtf8 ("text")));
text1.open (QIODevice::ReadOnly);
ba = text1.read (10000);
EXPECT_EQ (ba.constData (), "A text.\n");
text1.close ();
QFile text21 (dir21.absoluteFilePath (QString::fromUtf8 ("text")));
text21.open (QIODevice::ReadOnly);
ba = text21.read (10000);
EXPECT_EQ (ba.constData (), "A text II.I.\n");
text21.close ();
}

View File

@ -98,7 +98,8 @@ SOURCES = \
tlFileSystemWatcher.cc \
laySalt.cc \
tlFileUtils.cc \
tlHttpStream.cc
tlHttpStream.cc \
tlWebDAV.cc
# main components:
SOURCES += \