WIP on Salt package manager

- made WebDAV implementation compatible with Git's SVN API
- bugfix: parallel requests were not supported on tl::HttpStream

src/lay/laySaltGrain.cc

@@ -24,7 +24,9 @@
 #include "tlString.h"
 #include "tlXMLParser.h"
 #include "tlHttpStream.h"
+#include "tlWebDAV.h"
 
+#include <memory>
 #include <QDir>
 #include <QFileInfo>
 #include <QBuffer>
@@ -440,10 +442,17 @@ SaltGrain::from_url (const std::string &url)
     throw tl::Exception (tl::to_string (QObject::tr ("No download link available")));
   }
 
-  tl::InputStream stream (SaltGrain::spec_url (url));
+  std::auto_ptr<tl::InputStream> stream;
+  std::string spec_url = SaltGrain::spec_url (url);
+
+  if (spec_url.find ("http:") == 0 || spec_url.find ("https:") == 0) {
+    stream.reset (tl::WebDAVObject::download_item (spec_url));
+  } else {
+    stream.reset (new tl::InputStream (spec_url));
+  }
 
   SaltGrain g;
-  g.load (stream);
+  g.load (*stream);
   g.set_url (url);
   return g;
 }

src/lay/laySaltManagerDialog.cc

@@ -536,7 +536,9 @@ BEGIN_PROTECTED
     } else {
       unmark_all_new ();
     }
-    manager.execute (*mp_salt);
+    if (! manager.execute (*mp_salt)) {
+      throw tl::Exception (tl::to_string (tr ("Failed to install some of the selected packages. Please see log for details.")));
+    }
   }
 
 END_PROTECTED
@@ -863,6 +865,7 @@ void
 SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTextWidget *details)
 {
   if (! g) {
+    details->setHtml (QString ());
     return;
   }
 
@@ -891,11 +894,7 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex
 
     QApplication::processEvents (QEventLoop::ExcludeUserInputEvents);
 
-    tl::InputStream stream (SaltGrain::spec_url (g->url ()));
+    remote_grain.reset (new SaltGrain (SaltGrain::from_url (g->url ())));
-
-    remote_grain.reset (new SaltGrain ());
-    remote_grain->load (stream);
-    remote_grain->set_url (g->url ());
 
     if (g->name () != remote_grain->name ()) {
       throw tl::Exception (tl::to_string (tr ("Name mismatch between repository and actual package (repository: %1, package: %2)").arg (tl::to_qstring (g->name ())).arg (tl::to_qstring (remote_grain->name ()))));

src/tl/tlHttpStream.cc

@@ -72,28 +72,56 @@ public:
   }
 };
 
+// ---------------------------------------------------------------
+//  AuthenticationHandler implementation
+
+AuthenticationHandler::AuthenticationHandler ()
+  : QObject (0)
+{
+  //  .. nothing yet ..
+}
+
+void
+AuthenticationHandler::authenticationRequired (QNetworkReply *reply, QAuthenticator *auth)
+{
+  PasswordDialog pw_dialog (0 /*no parent*/);
+  pw_dialog.exec_auth (false, reply->url ().toString (), auth);
+}
+
+void
+AuthenticationHandler::proxyAuthenticationRequired (const QNetworkProxy &proxy, QAuthenticator *auth)
+{
+  PasswordDialog pw_dialog (0 /*no parent*/);
+  pw_dialog.exec_auth (true, proxy.hostName (), auth);
+}
+
 // ---------------------------------------------------------------
 //  InputHttpFile implementation
 
 static QNetworkAccessManager *s_network_manager (0);
+static AuthenticationHandler *s_auth_handler (0);
 
 InputHttpStream::InputHttpStream (const std::string &url)
-  : m_url (url), m_request ("GET"), mp_buffer (0)
+  : m_url (url), m_request ("GET"), mp_buffer (0), mp_reply (0)
 {
   if (! s_network_manager) {
-    s_network_manager = new QNetworkAccessManager(0);
+
+    s_network_manager = new QNetworkAccessManager (0);
+    s_auth_handler = new AuthenticationHandler ();
+    connect (s_network_manager, SIGNAL (authenticationRequired (QNetworkReply *, QAuthenticator *)), s_auth_handler, SLOT (authenticationRequired (QNetworkReply *, QAuthenticator *)));
+    connect (s_network_manager, SIGNAL (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)), s_auth_handler, SLOT (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)));
+
     tl::StaticObjects::reg (&s_network_manager);
+    tl::StaticObjects::reg (&s_auth_handler);
+
   }
+
   connect (s_network_manager, SIGNAL (finished (QNetworkReply *)), this, SLOT (finished (QNetworkReply *)));
-  connect (s_network_manager, SIGNAL (authenticationRequired (QNetworkReply *, QAuthenticator *)), this, SLOT (authenticationRequired (QNetworkReply *, QAuthenticator *)));
-  connect (s_network_manager, SIGNAL (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)), this, SLOT (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)));
-  mp_reply = 0;
 }
 
 InputHttpStream::~InputHttpStream ()
 {
-  delete mp_reply;
+  //  .. nothing yet ..
-  mp_reply = 0;
 }
 
 void
@@ -120,28 +148,17 @@ InputHttpStream::add_header (const std::string &name, const std::string &value)
   m_headers.insert (std::make_pair (name, value));
 }
 
-void 
-InputHttpStream::authenticationRequired (QNetworkReply *reply, QAuthenticator *auth)
-{
-  PasswordDialog pw_dialog (0 /*no parent*/);
-  pw_dialog.exec_auth (false, reply->url ().toString (), auth);
-}
-
-void 
-InputHttpStream::proxyAuthenticationRequired (const QNetworkProxy &proxy, QAuthenticator *auth)
-{
-  PasswordDialog pw_dialog (0 /*no parent*/);
-  pw_dialog.exec_auth (true, proxy.hostName (), auth);
-}
-
 void 
 InputHttpStream::finished (QNetworkReply *reply)
 {
+  if (reply != mp_active_reply.get ()) {
+    return;
+  }
+
   QVariant redirect_target = reply->attribute (QNetworkRequest::RedirectionTargetAttribute);
   if (reply->error () == QNetworkReply::NoError && ! redirect_target.isNull ()) {
     m_url = tl::to_string (redirect_target.toString ());
     issue_request (QUrl (redirect_target.toString ()));
-    delete reply;
   } else {
     mp_reply = reply;
   }
@@ -158,10 +175,10 @@ InputHttpStream::issue_request (const QUrl &url)
     request.setRawHeader (QByteArray (h->first.c_str ()), QByteArray (h->second.c_str ()));
   }
   if (m_data.isEmpty ()) {
-    s_network_manager->sendCustomRequest (request, m_request);
+    mp_active_reply.reset (s_network_manager->sendCustomRequest (request, m_request));
   } else {
     mp_buffer = new QBuffer (&m_data);
-    s_network_manager->sendCustomRequest (request, m_request, mp_buffer);
+    mp_active_reply.reset (s_network_manager->sendCustomRequest (request, m_request, mp_buffer));
   }
 }
 
@@ -171,6 +188,8 @@ InputHttpStream::read (char *b, size_t n)
   if (mp_reply == 0) {
     issue_request (QUrl (tl::to_qstring (m_url)));
   }
+
+  //  @@@ TODO: progress, timeout
   while (mp_reply == 0) {
     QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents, 100);
   }

src/tl/tlHttpStream.h

@@ -29,6 +29,7 @@
 #include <QObject>
 #include <QBuffer>
 #include <QByteArray>
+#include <memory>
 
 class QNetworkAccessManager;
 class QNetworkReply;
@@ -47,6 +48,19 @@ public:
   { }
 };
 
+class AuthenticationHandler
+  : public QObject
+{
+Q_OBJECT
+
+public:
+  AuthenticationHandler ();
+
+public slots:
+  void authenticationRequired (QNetworkReply *, QAuthenticator *);
+  void proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *);
+};
+
 /**
  *  @brief A http input delegate for tl::InputStream
  *
@@ -117,12 +131,11 @@ public:
 
 private slots:
   void finished (QNetworkReply *);
-  void authenticationRequired (QNetworkReply *, QAuthenticator *);
-  void proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *);
 
 private:
   std::string m_url;
   QNetworkReply *mp_reply;
+  std::auto_ptr<QNetworkReply> mp_active_reply;
   QByteArray m_request;
   QByteArray m_data;
   QBuffer *mp_buffer;

src/tl/tlStream.cc

@@ -468,6 +468,17 @@ InputStream::read_all ()
   return str;
 }
 
+void
+InputStream::copy_to (tl::OutputStream &os)
+{
+  const size_t chunk = 65536;
+  char b [chunk];
+  size_t read;
+  while ((read = mp_delegate->read (b, sizeof (b))) > 0) {
+    os.put (b, read);
+  }
+}
+
 void
 InputStream::inflate ()
 {

src/tl/tlStream.h

@@ -41,6 +41,7 @@ namespace tl
 
 class InflateFilter;
 class DeflateFilter;
+class OutputStream;
 
 // ---------------------------------------------------------------------------------
 
@@ -230,6 +231,11 @@ public:
    */
   std::string read_all (size_t max_count);
 
+  /**
+   *  @brief Copies the content of the stream to the output stream
+   */
+  void copy_to (tl::OutputStream &os);
+
   /**
    *  @brief Enable uncompression of the following DEFLATE-compressed block
    *

src/tl/tlWebDAV.cc

@@ -29,6 +29,7 @@
 #include "tlProgress.h"
 #include "tlLog.h"
 
+#include <memory>
 #include <QUrl>
 #include <QDir>
 
@@ -136,16 +137,11 @@ static std::string item_name (const QString &path1, const QString &path2)
     sl2.pop_back ();
   }
 
-  int i = 0;
+  if (sl1 == sl2) {
-  for ( ; i < sl1.length () && i < sl2.length (); ++i) {
+    //  This is the top-level item (echoed in the PROPFIND response)
-    if (sl1 [i] != sl2 [i]) {
-      throw tl::Exception (tl::to_string (QObject::tr ("Invalid WebDAV response: %1 is not a collection item of %2").arg (path2).arg (path1)));
-    }
-  }
-  if (i == sl2.length ()) {
     return std::string ();
-  } else if (i + 1 == sl2.length ()) {
+  } else if (! sl2.empty ()) {
-    return tl::to_string (sl2[i]);
+    return tl::to_string (sl2.back ());
   } else {
     throw tl::Exception (tl::to_string (QObject::tr ("Invalid WebDAV response: %1 is not a collection sub-item of %2").arg (path2).arg (path1)));
   }
@@ -254,6 +250,15 @@ void fetch_download_items (const std::string &url, const std::string &target, st
   }
 }
 
+tl::InputStream *
+WebDAVObject::download_item (const std::string &url)
+{
+  tl::InputHttpStream *http = new tl::InputHttpStream (url);
+  //  This trick allows accessing GitHub repos through their SVN API
+  http->add_header ("User-Agent", "SVN");
+  return new tl::InputStream (*http);
+}
+
 bool
 WebDAVObject::download (const std::string &url, const std::string &target)
 {
@@ -283,26 +288,9 @@ WebDAVObject::download (const std::string &url, const std::string &target)
 
       try {
 
-        tl::InputHttpStream http (i->url);
+        tl::OutputStream os (i->path);
-
+        std::auto_ptr<tl::InputStream> is (download_item (i->url));
-        QFile file (tl::to_qstring (i->path));
+        is->copy_to (os);
-        if (! file.open (QIODevice::WriteOnly)) {
-          has_errors = true;
-          tl::error << QObject::tr ("Unable to open file '%1' for writing").arg (tl::to_qstring (i->path));
-        }
-
-        const size_t chunk = 65536;
-        char b[chunk];
-        size_t read;
-        while ((read = http.read (b, sizeof (b))) > 0) {
-          if (! file.write (b, read)) {
-            tl::error << QObject::tr ("Unable to write %2 bytes file '%1'").arg (tl::to_qstring (i->path)).arg (int (read));
-            has_errors = true;
-            break;
-          }
-        }
-
-        file.close ();
 
       } catch (tl::Exception &ex) {
         tl::error << QObject::tr ("Error downloading file from '") << i->url << "':" << tl::endl << ex.msg ();

src/tl/tlWebDAV.h

@@ -25,6 +25,8 @@
 #define HDR_tlWebDAV
 
 #include "tlCommon.h"
+#include "tlStream.h"
+
 #include <string>
 #include <vector>
 
@@ -142,6 +144,13 @@ public:
    */
   static bool download (const std::string &url, const std::string &target);
 
+  /**
+   *  @brief Gets a stream object for downloading the single item of the given URL
+   *
+   *  The stream object returned needs to be deleted by the caller.
+   */
+  static tl::InputStream *download_item (const std::string &url);
+
 private:
   container m_items;
 };