luke
/
klayout
mirror of https://github.com/KLayout/klayout.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'matthias-origin/async-reader' into macos-build-on-master

This commit is contained in:
Matthias Koefferlein 2018-01-06 17:10:35 -08:00
parent 8003d1bb47 8eb3f5e6ce
commit 55be7d78e7
10 changed files with 277 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/lay/lay/laySaltGrain.cc
View File

@ -480,15 +480,14 @@ SaltGrain::from_path (const std::string &path)
return g;
}
SaltGrain
SaltGrain::from_url (const std::string &url_in)
tl::InputStream *
SaltGrain::stream_from_url (std::string &url_in)
{
if (url_in.empty ()) {
throw tl::Exception (tl::to_string (QObject::tr ("No download link available")));
}
std::string url = url_in;
std::auto_ptr<tl::InputStream> stream;
// base relative URL's on the salt mine URL
if (url.find ("http:") != 0 && url.find ("https:") != 0 && url.find ("file:") != 0 && !url.empty() && url[0] != '/' && url[0] != '\\' && lay::SaltController::instance ()) {
@ -501,16 +500,23 @@ SaltGrain::from_url (const std::string &url_in)
}
sami_url.setPath (path_comp.join (QString::fromUtf8 ("/")));
url = tl::to_string (sami_url.toString ());
url_in = tl::to_string (sami_url.toString ());
}
std::string spec_url = SaltGrain::spec_url (url);
if (spec_url.find ("http:") == 0 || spec_url.find ("https:") == 0) {
stream.reset (tl::WebDAVObject::download_item (spec_url));
return tl::WebDAVObject::download_item (spec_url);
} else {
stream.reset (new tl::InputStream (spec_url));
return new tl::InputStream (spec_url);
}
}
SaltGrain
SaltGrain::from_url (const std::string &url_in)
{
std::string url = url_in;
std::auto_ptr<tl::InputStream> stream (stream_from_url (url));
SaltGrain g;
g.load (*stream);

8
src/lay/lay/laySaltGrain.h
View File

@ -449,6 +449,14 @@ public:
*/
static SaltGrain from_url (const std::string &url);
/**
* @brief Returns a stream prepared for downloading the grain
* The stream is a new'd object and needs to be deleted by the caller.
* "url" is the download URL on input and gets modified to match the
* actual URL if it is a relative one.
*/
static tl::InputStream *stream_from_url (std::string &url);
/**
* @brief Forms the spec file download URL from a given download URL
*/

195
src/lay/lay/laySaltManagerDialog.cc
View File

@ -113,7 +113,8 @@ private:
SaltManagerDialog::SaltManagerDialog (QWidget *parent, lay::Salt *salt, const std::string &salt_mine_url)
: QDialog (parent),
m_salt_mine_url (salt_mine_url),
dm_update_models (this, &SaltManagerDialog::update_models), m_current_tab (-1)
dm_update_models (this, &SaltManagerDialog::update_models), m_current_tab (-1),
mp_downloaded_target (0)
{
Ui::SaltManagerDialog::setupUi (this);
mp_properties_dialog = new lay::SaltGrainPropertiesDialog (this);
@ -126,18 +127,6 @@ SaltManagerDialog::SaltManagerDialog (QWidget *parent, lay::Salt *salt, const st
mp_salt = salt;
QApplication::setOverrideCursor (Qt::WaitCursor);
try {
if (! m_salt_mine_url.empty ()) {
tl::log << tl::to_string (tr ("Downloading package repository from %1").arg (tl::to_qstring (m_salt_mine_url)));
m_salt_mine.load (m_salt_mine_url);
}
m_salt_mine.consolidate ();
} catch (tl::Exception &ex) {
tl::error << ex.msg ();
}
QApplication::restoreOverrideCursor ();
SaltModel *model = new SaltModel (this, mp_salt);
model->set_empty_explanation (tr ("No packages are present on this system"));
salt_view->setModel (model);
@ -234,7 +223,7 @@ SaltManagerDialog::SaltManagerDialog (QWidget *parent, lay::Salt *salt, const st
connect (actionMarkForUpdate, SIGNAL (triggered ()), this, SLOT (mark_clicked ()));
connect (actionUnmarkForUpdate, SIGNAL (triggered ()), this, SLOT (mark_clicked ()));
update_models ();
refresh ();
}
void
@ -245,10 +234,13 @@ SaltManagerDialog::mode_changed ()
QList<int> sizes;
if (m_current_tab == 2) {
selected_changed ();
sizes = splitter->sizes ();
} else if (m_current_tab == 1) {
mine_update_selected_changed ();
sizes = splitter_update->sizes ();
} else if (m_current_tab == 0) {
mine_new_selected_changed ();
sizes = splitter_new->sizes ();
}
@ -268,6 +260,7 @@ SaltManagerDialog::mode_changed ()
}
m_current_tab = mode_tab->currentIndex ();
update_apply_state ();
}
void
@ -437,7 +430,7 @@ SaltManagerDialog::update_apply_state ()
{
SaltModel *model;
model = dynamic_cast <SaltModel *> (salt_mine_view_new->model ());
model = dynamic_cast <SaltModel *> (salt_mine_view_new->model ());
if (model) {
int marked = 0;
@ -665,30 +658,61 @@ SaltManagerDialog::salt_mine_about_to_change ()
void
SaltManagerDialog::refresh ()
{
BEGIN_PROTECTED
if (! m_salt_mine_url.empty ()) {
tl::log << tl::to_string (tr ("Downloading package repository from %1").arg (tl::to_qstring (m_salt_mine_url)));
try {
m_salt_mine_reader.reset (new tl::InputStream (m_salt_mine_url));
salt_mine_download_started ();
QApplication::setOverrideCursor (Qt::WaitCursor);
lay::Salt new_mine;
new_mine.load (m_salt_mine_url);
m_salt_mine = new_mine;
QApplication::restoreOverrideCursor ();
} catch (...) {
QApplication::restoreOverrideCursor ();
throw;
tl::InputHttpStream *http = dynamic_cast<tl::InputHttpStream *> (m_salt_mine_reader->base ());
if (http) {
// async reading on HTTP
http->ready ().add (this, &SaltManagerDialog::salt_mine_data_ready);
http->send ();
} else {
salt_mine_data_ready ();
}
salt_mine_changed ();
}
}
void
SaltManagerDialog::salt_mine_download_started ()
{
QApplication::setOverrideCursor (Qt::WaitCursor);
}
void
SaltManagerDialog::salt_mine_download_finished ()
{
QApplication::restoreOverrideCursor ();
m_salt_mine_reader.reset (0);
}
void
SaltManagerDialog::salt_mine_data_ready ()
{
BEGIN_PROTECTED
try {
if (m_salt_mine_reader.get ()) {
lay::Salt new_mine;
new_mine.load (*m_salt_mine_reader);
m_salt_mine = new_mine;
}
salt_mine_download_finished ();
} catch (...) {
salt_mine_download_finished ();
throw;
}
salt_mine_changed ();
END_PROTECTED
}
@ -824,10 +848,7 @@ SaltManagerDialog::update_models ()
salt_mine_view_new->selectionModel ()->blockSignals (false);
}
mine_new_selected_changed ();
mine_update_selected_changed ();
selected_changed ();
update_apply_state ();
mode_changed ();
}
void
@ -929,13 +950,14 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex
return;
}
std::auto_ptr<lay::SaltGrain> remote_grain;
m_downloaded_grain.reset (0);
m_downloaded_grain_reader.reset (0);
mp_downloaded_target = details;
m_salt_mine_grain.reset (new lay::SaltGrain (*g));
// Download actual grain definition file
try {
QApplication::setOverrideCursor (Qt::WaitCursor);
if (g->url ().empty ()) {
throw tl::Exception (tl::to_string (tr ("No download link available")));
}
@ -954,44 +976,75 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex
details->setHtml (html);
QApplication::processEvents (QEventLoop::ExcludeUserInputEvents);
std::string url = g->url ();
m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url));
m_downloaded_grain.reset (new SaltGrain ());
m_downloaded_grain->set_url (url);
remote_grain.reset (new SaltGrain (SaltGrain::from_url (g->url ())));
if (g->name () != remote_grain->name ()) {
throw tl::Exception (tl::to_string (tr ("Name mismatch between repository and actual package (repository: %1, package: %2)").arg (tl::to_qstring (g->name ())).arg (tl::to_qstring (remote_grain->name ()))));
tl::InputHttpStream *http = dynamic_cast<tl::InputHttpStream *> (m_downloaded_grain_reader->base ());
if (http) {
// async reading on HTTP
http->ready ().add (this, &SaltManagerDialog::data_ready);
http->send ();
} else {
data_ready ();
}
if (SaltGrain::compare_versions (g->version (), remote_grain->version ()) != 0) {
throw tl::Exception (tl::to_string (tr ("Version mismatch between repository and actual package (repository: %1, package: %2)").arg (tl::to_qstring (g->version ())).arg (tl::to_qstring (remote_grain->version ()))));
}
details->set_grain (remote_grain.get ());
QApplication::restoreOverrideCursor ();
} catch (tl::Exception &ex) {
QApplication::restoreOverrideCursor ();
remote_grain.reset (0);
QString html = tr (
"<html>"
"<body>"
"<font color=\"#ff0000\">"
"<h2>Error Fetching Package Definition</h2>"
"<p><b>URL</b>: %1</p>"
"<p><b>Error</b>: %2</p>"
"</font>"
"</body>"
"</html>"
)
.arg (tl::to_qstring (SaltGrain::spec_url (g->url ())))
.arg (tl::to_qstring (tl::escaped_to_html (ex.msg ())));
details->setHtml (html);
show_error (ex);
}
}
void
SaltManagerDialog::data_ready ()
{
if (! m_salt_mine_grain.get () || ! m_downloaded_grain.get () || ! m_downloaded_grain_reader.get () || ! mp_downloaded_target) {
return;
}
m_downloaded_grain->load (*m_downloaded_grain_reader);
try {
if (m_salt_mine_grain->name () != m_downloaded_grain->name ()) {
throw tl::Exception (tl::to_string (tr ("Name mismatch between repository and actual package (repository: %1, package: %2)").arg (tl::to_qstring (m_salt_mine_grain->name ())).arg (tl::to_qstring (m_downloaded_grain->name ()))));
}
if (SaltGrain::compare_versions (m_salt_mine_grain->version (), m_downloaded_grain->version ()) != 0) {
throw tl::Exception (tl::to_string (tr ("Version mismatch between repository and actual package (repository: %1, package: %2)").arg (tl::to_qstring (m_salt_mine_grain->version ())).arg (tl::to_qstring (m_downloaded_grain->version ()))));
}
mp_downloaded_target->set_grain (m_downloaded_grain.get ());
m_downloaded_grain.reset (0);
m_downloaded_grain_reader.reset (0);
m_salt_mine_grain.reset (0);
} catch (tl::Exception &ex) {
show_error (ex);
}
}
void
SaltManagerDialog::show_error (tl::Exception &ex)
{
QString html = tr (
"<html>"
"<body>"
"<font color=\"#ff0000\">"
"<h2>Error Fetching Package Definition</h2>"
"<p><b>URL</b>: %1</p>"
"<p><b>Error</b>: %2</p>"
"</font>"
"</body>"
"</html>"
)
.arg (tl::to_qstring (m_downloaded_grain->url ()))
.arg (tl::to_qstring (tl::escaped_to_html (ex.msg ())));
mp_downloaded_target->setHtml (html);
m_downloaded_grain.reset (0);
m_downloaded_grain_reader.reset (0);
m_salt_mine_grain.reset (0);
}
}

22
src/lay/lay/laySaltManagerDialog.h
View File

@ -26,6 +26,8 @@
#include "ui_SaltManagerDialog.h"
#include "laySalt.h"
#include "tlDeferredExecution.h"
#include "tlHttpStream.h"
#include "tlException.h"
#include <QDialog>
#include <memory>
@ -40,7 +42,7 @@ class SaltGrainPropertiesDialog;
* @brief The dialog for managing the Salt ("Packages")
*/
class SaltManagerDialog
: public QDialog, private Ui::SaltManagerDialog
: public QDialog, private Ui::SaltManagerDialog, public tl::Object
{
Q_OBJECT
@ -58,6 +60,17 @@ public:
return m_salt_mine_url;
}
private:
/**
* @brief Called when data is available from the grain downloader
*/
void data_ready ();
/**
* @brief Called when data is available from the salt mine downloader
*/
void salt_mine_data_ready ();
private slots:
/**
* @brief Called when the list of packages (grains) is about to change
@ -171,6 +184,10 @@ private:
SaltGrainPropertiesDialog *mp_properties_dialog;
tl::DeferredMethod<SaltManagerDialog> dm_update_models;
int m_current_tab;
std::auto_ptr<tl::InputStream> m_downloaded_grain_reader;
std::auto_ptr<lay::SaltGrain> m_downloaded_grain, m_salt_mine_grain;
SaltGrainDetailsTextWidget *mp_downloaded_target;
std::auto_ptr<tl::InputStream> m_salt_mine_reader;
SaltGrain *current_grain ();
std::vector<lay::SaltGrain *> current_grains ();
@ -179,6 +196,9 @@ private:
void update_apply_state ();
void get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTextWidget *details);
void consolidate_salt_mine_entries ();
void show_error (tl::Exception &ex);
void salt_mine_download_started ();
void salt_mine_download_finished ();
};
}

11
src/tl/tl/tlHttpStream.cc
View File

@ -123,7 +123,7 @@ InputHttpStream::InputHttpStream (const std::string &url)
InputHttpStream::~InputHttpStream ()
{
// .. nothing yet ..
// .. nothing yet ..
}
void
@ -166,6 +166,7 @@ InputHttpStream::finished (QNetworkReply *reply)
issue_request (QUrl (redirect_target.toString ()));
} else {
mp_reply = reply;
m_ready ();
}
}
@ -205,6 +206,14 @@ InputHttpStream::issue_request (const QUrl &url)
#endif
}
void
InputHttpStream::send ()
{
if (mp_reply == 0) {
issue_request (QUrl (tl::to_qstring (m_url)));
}
}
size_t
InputHttpStream::read (char *b, size_t n)
{

32
src/tl/tl/tlHttpStream.h
View File

@ -25,6 +25,7 @@
#define HDR_tlHttpStream
#include "tlStream.h"
#include "tlEvents.h"
#include <QObject>
#include <QBuffer>
@ -84,6 +85,19 @@ public:
*/
virtual ~InputHttpStream ();
/**
* @brief Sends the request for data
* To ensure prompt delivery of data, this method can be used prior to
* "read" to trigger the download from the given URL.
* This method will return immediately. When the reply is available,
* the "ready" event will be triggered. "read" can then be used to
* read the data or - in case of an error - throw an exception.
* If "send" is not used before "read", "read" will block until data
* is available.
* If a request has already been sent, this method will do nothing.
*/
void send ();
/**
* @brief Sets the request verb
* The default verb is "GET"
@ -115,6 +129,23 @@ public:
*/
virtual size_t read (char *b, size_t n);
/**
* @brief Gets the "ready" event
* Connect to this event for the asynchroneous interface.
*/
tl::Event &ready ()
{
return m_ready;
}
/**
* @brief Gets a value indicating whether data is available
*/
bool data_available ()
{
return mp_reply != 0;
}
virtual void reset ();
virtual std::string source () const
@ -140,6 +171,7 @@ private:
QByteArray m_data;
QBuffer *mp_buffer;
std::map<std::string, std::string> m_headers;
tl::Event m_ready;
void issue_request (const QUrl &url);
};

18
src/tl/tl/tlStream.cc
View File

@ -311,6 +311,14 @@ InputStream::InputStream (InputStreamBase &delegate)
mp_buffer = new char [m_bcap];
}
InputStream::InputStream (InputStreamBase *delegate)
: m_pos (0), mp_bptr (0), mp_delegate (delegate), m_owns_delegate (true), mp_inflate (0)
{
m_bcap = 4096; // initial buffer capacity
m_blen = 0;
mp_buffer = new char [m_bcap];
}
InputStream::InputStream (const std::string &abstract_path)
: m_pos (0), mp_bptr (0), mp_delegate (0), m_owns_delegate (false), mp_inflate (0)
{
@ -357,7 +365,15 @@ std::string InputStream::absolute_path (const std::string &abstract_path)
InputStream::~InputStream ()
{
if (mp_delegate && m_owns_delegate) {
delete mp_delegate;
// NOTE: HTTP stream objects should not be deleted now, since events
// may be pending that deliver the finished signal to the object.
tl::InputHttpStream *http = dynamic_cast<tl::InputHttpStream *>(mp_delegate);
if (http) {
http->ready ().clear (); // avoids events from deleted streams
http->deleteLater ();
} else {
delete mp_delegate;
}
mp_delegate = 0;
}
if (mp_inflate) {

17
src/tl/tl/tlStream.h
View File

@ -172,11 +172,16 @@ class TL_PUBLIC InputStream
public:
/**
* @brief Default constructor
*
* This constructor takes a delegate object.
* This constructor takes a delegate object, but does not take ownership.
*/
InputStream (InputStreamBase &delegate);
/**
* @brief Default constructor
* This constructor takes a delegate object, and takes ownership.
*/
InputStream (InputStreamBase *delegate);
/**
* @brief Opens a stream from a abstract path
*
@ -305,6 +310,14 @@ public:
* @brief Gets the absolute path for a given URL
*/
static std::string absolute_path (const std::string &path);
/**
* @brief Gets the base reader (delegate)
*/
InputStreamBase *base ()
{
return mp_delegate;
}
protected:
void reset_pos ()

2
src/tl/tl/tlWebDAV.cc
View File

@ -256,7 +256,7 @@ WebDAVObject::download_item (const std::string &url)
tl::InputHttpStream *http = new tl::InputHttpStream (url);
// This trick allows accessing GitHub repos through their SVN API
http->add_header ("User-Agent", "SVN");
return new tl::InputStream (*http);
return new tl::InputStream (http);
}
bool

37
src/tl/unit_tests/tlHttpStream.cc
View File

@ -24,6 +24,8 @@
#include "tlHttpStream.h"
#include "tlUnitTest.h"
#include <QCoreApplication>
static std::string test_url1 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/text");
static std::string test_url2 ("http://www.klayout.org/svn-public/klayout-resources/trunk/testdata/dir1");
@ -73,3 +75,38 @@ TEST(2)
"</D:multistatus>\n"
);
}
namespace
{
class Receiver : public tl::Object
{
public:
Receiver () : flag (false) { }
void handle () { flag = true; }
bool flag;
};
}
// async mode
TEST(3)
{
tl::InputHttpStream stream (test_url1);
Receiver r;
stream.ready ().add (&r, &Receiver::handle);
stream.send ();
EXPECT_EQ (stream.data_available (), false);
while (! stream.data_available ()) {
QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents);
}
EXPECT_EQ (r.flag, true);
char b[100];
size_t n = stream.read (b, sizeof (b));
std::string res (b, n);
EXPECT_EQ (res, "hello, world.\n");
}