diff --git a/src/klayout.pri b/src/klayout.pri index 28c1f6cbc..36af52144 100644 --- a/src/klayout.pri +++ b/src/klayout.pri @@ -43,6 +43,11 @@ equals(HAVE_PYTHON, "1") { DEFINES += HAVE_PYTHON } +equals(HAVE_CURL, "1") { + DEFINES += HAVE_CURL + LIBS += -lcurl +} + equals(HAVE_RUBY, "1") { DEFINES += \ HAVE_RUBY \ diff --git a/src/tl/tl/PasswordDialog.ui b/src/lay/lay/PasswordDialog.ui similarity index 100% rename from src/tl/tl/PasswordDialog.ui rename to src/lay/lay/PasswordDialog.ui diff --git a/src/lay/lay/lay.pro b/src/lay/lay/lay.pro index bc8e16865..8a1ff4a69 100644 --- a/src/lay/lay/lay.pro +++ b/src/lay/lay/lay.pro @@ -57,7 +57,8 @@ HEADERS = \ layFontController.h \ layNativePlugin.h \ laySystemPaths.h \ - layMacroEditorSetupPage.h + layMacroEditorSetupPage.h \ + layPasswordDialog.h FORMS = \ ClipDialog.ui \ @@ -105,7 +106,8 @@ FORMS = \ SaltGrainTemplateSelectionDialog.ui \ SaltManagerInstallConfirmationDialog.ui \ CustomizeMenuConfigPage.ui \ - MacroEditorSetupPage.ui + MacroEditorSetupPage.ui \ + PasswordDialog.ui SOURCES = \ gsiDeclLayApplication.cc \ @@ -160,7 +162,8 @@ SOURCES = \ layFontController.cc \ layNativePlugin.cc \ laySystemPaths.cc \ - layMacroEditorSetupPage.cc + layMacroEditorSetupPage.cc \ + layPasswordDialog.cc RESOURCES = layBuildInMacros.qrc \ layHelpResources.qrc \ diff --git a/src/lay/lay/layApplication.cc b/src/lay/lay/layApplication.cc index 56a3651ca..556994445 100644 --- a/src/lay/lay/layApplication.cc +++ b/src/lay/lay/layApplication.cc @@ -38,6 +38,7 @@ #include "layTechnologyController.h" #include "laySaltController.h" #include "laySystemPaths.h" +#include "layPasswordDialog.h" #include "lymMacro.h" #include "gtf.h" #include "gsiDecl.h" @@ -56,6 +57,7 @@ #include "tlExpression.h" #include "tlExceptions.h" #include "tlInternational.h" +#include "tlHttpStream.h" #include "tlArch.h" #include @@ -1520,6 +1522,10 @@ GuiApplication::setup () mp_mw = new lay::MainWindow (this, "main_window"); QObject::connect (mp_mw, SIGNAL (closed ()), this, SLOT (quit ())); + + // create a password dialog for use with the HTTP streams + lay::PasswordDialog *pw_dialog = new lay::PasswordDialog (mp_mw); + tl::InputHttpStream::set_credential_provider (pw_dialog); } void diff --git a/src/lay/lay/layPasswordDialog.cc b/src/lay/lay/layPasswordDialog.cc new file mode 100644 index 000000000..dfc6341d9 --- /dev/null +++ b/src/lay/lay/layPasswordDialog.cc @@ -0,0 +1,62 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2018 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "layPasswordDialog.h" + +namespace lay +{ + +PasswordDialog::PasswordDialog (QWidget *parent) + : QDialog (parent) +{ + setupUi (this); +} + +bool +PasswordDialog::user_password (const std::string &url, const std::string &realm, bool proxy, int attempt, std::string &user, std::string &passwd) +{ + realm_label->setText (tr ("Realm: ") + tl::to_qstring (realm)); + if (proxy) { + where_label->setText (tr ("Proxy: ") + tl::to_qstring (url)); + } else { + where_label->setText (tr ("URL: ") + tl::to_qstring (url)); + } + + if (attempt > 1) { + attempt_label->setText (tr ("Authentication failed - please try again")); + attempt_label->show (); + } else { + attempt_label->hide (); + } + + if (QDialog::exec ()) { + passwd = tl::to_string (password_le->text ()); + user = tl::to_string (user_le->text ()); + return true; + } else { + return false; + } + +} + +} + diff --git a/src/lay/lay/layPasswordDialog.h b/src/lay/lay/layPasswordDialog.h new file mode 100644 index 000000000..7b20d4fbf --- /dev/null +++ b/src/lay/lay/layPasswordDialog.h @@ -0,0 +1,50 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2018 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef HDR_layPasswordDialog +#define HDR_layPasswordDialog + +#include "tlHttpStream.h" + +#include "ui_PasswordDialog.h" +#include + +namespace lay +{ + +/** + * @brief A password dialog for registration with tl::HttpStream + */ +class PasswordDialog + : public QDialog, public tl::HttpCredentialProvider, private Ui::PasswordDialog +{ +public: + PasswordDialog (QWidget *parent); + + bool user_password (const std::string &url, const std::string &realm, bool proxy, int attempt, std::string &user, std::string &passwd); +}; + + + +} + +#endif diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 48595c33c..ff2d8d61b 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -8,8 +8,7 @@ DEFINES += MAKE_TL_LIBRARY LIBS += -lz -FORMS = \ - PasswordDialog.ui +FORMS = SOURCES = \ tlAssert.cc \ @@ -43,7 +42,9 @@ SOURCES = \ tlArch.cc \ tlCommandLineParser.cc \ tlUnitTest.cc \ - tlInt128Support.cc + tlInt128Support.cc \ + tlHttpStreamCurl.cc \ + tlHttpStreamQt.cc HEADERS = \ tlAlgorithm.h \ @@ -92,7 +93,9 @@ HEADERS = \ tlArch.h \ tlCommandLineParser.h \ tlUnitTest.h \ - tlInt128Support.h + tlInt128Support.h \ + tlHttpStreamCurl.h \ + tlHttpStreamQt.h INCLUDEPATH = DEPENDPATH = diff --git a/src/tl/tl/tlHttpStream.cc b/src/tl/tl/tlHttpStream.cc index 87069b588..b2268cca8 100644 --- a/src/tl/tl/tlHttpStream.cc +++ b/src/tl/tl/tlHttpStream.cc @@ -22,287 +22,10 @@ #include "tlHttpStream.h" -#include "tlLog.h" -#include "tlStaticObjects.h" -#include "tlDeferredExecution.h" - -#include "ui_PasswordDialog.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include namespace tl { -// --------------------------------------------------------------- -// PasswordDialog definition and implementation - -class PasswordDialog - : public QDialog, private Ui::PasswordDialog -{ -public: - PasswordDialog (QWidget *parent) - : QDialog (parent) - { - setupUi (this); - } - - bool exec_auth (bool proxy, int attempt, const QString &where, QAuthenticator *auth) - { - realm_label->setText (tr ("Realm: ") + auth->realm ()); - if (proxy) { - where_label->setText (tr ("Proxy: ") + where); - } else { - where_label->setText (tr ("URL: ") + where); - } - - if (attempt > 1) { - attempt_label->setText (tr ("Authentication failed - please try again")); - attempt_label->show (); - } else { - attempt_label->hide (); - } - - if (QDialog::exec ()) { - auth->setPassword (password_le->text ()); - auth->setUser (user_le->text ()); - return true; - } else { - return false; - } - - } -}; - -// --------------------------------------------------------------- -// AuthenticationHandler implementation - -AuthenticationHandler::AuthenticationHandler () - : QObject (0), m_retry (0), m_proxy_retry (0) -{ // .. nothing yet .. -} - -void -AuthenticationHandler::reset () -{ - m_retry = 0; - m_proxy_retry = 0; -} - -void -AuthenticationHandler::authenticationRequired (QNetworkReply *reply, QAuthenticator *auth) -{ - PasswordDialog pw_dialog (0 /*no parent*/); - pw_dialog.exec_auth (false, ++m_retry, reply->url ().toString (), auth); -} - -void -AuthenticationHandler::proxyAuthenticationRequired (const QNetworkProxy &proxy, QAuthenticator *auth) -{ - PasswordDialog pw_dialog (0 /*no parent*/); - pw_dialog.exec_auth (true, ++m_proxy_retry, proxy.hostName (), auth); -} - -// --------------------------------------------------------------- -// InputHttpFile implementation - -static QNetworkAccessManager *s_network_manager (0); -static AuthenticationHandler *s_auth_handler (0); - -InputHttpStream::InputHttpStream (const std::string &url) - : m_url (url), mp_reply (0), m_request ("GET"), mp_buffer (0) -{ - if (! s_network_manager) { - - s_network_manager = new QNetworkAccessManager (0); - s_auth_handler = new AuthenticationHandler (); - connect (s_network_manager, SIGNAL (authenticationRequired (QNetworkReply *, QAuthenticator *)), s_auth_handler, SLOT (authenticationRequired (QNetworkReply *, QAuthenticator *))); - connect (s_network_manager, SIGNAL (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)), s_auth_handler, SLOT (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *))); - - tl::StaticObjects::reg (&s_network_manager); - tl::StaticObjects::reg (&s_auth_handler); - - } - - connect (s_network_manager, SIGNAL (finished (QNetworkReply *)), this, SLOT (finished (QNetworkReply *))); -} - -InputHttpStream::~InputHttpStream () -{ - // .. nothing yet .. -} - -void -InputHttpStream::set_request (const char *r) -{ - m_request = QByteArray (r); -} - -void -InputHttpStream::set_data (const char *data) -{ - m_data = QByteArray (data); -} - -void -InputHttpStream::set_data (const char *data, size_t n) -{ - m_data = QByteArray (data, int (n)); -} - -void -InputHttpStream::add_header (const std::string &name, const std::string &value) -{ - m_headers.insert (std::make_pair (name, value)); -} - -void -InputHttpStream::finished (QNetworkReply *reply) -{ - if (reply != mp_active_reply.get ()) { - return; - } - - QVariant redirect_target = reply->attribute (QNetworkRequest::RedirectionTargetAttribute); - if (reply->error () == QNetworkReply::NoError && ! redirect_target.isNull ()) { - m_url = tl::to_string (redirect_target.toString ()); - if (tl::verbosity() >= 30) { - tl::info << "HTTP redirect to: " << m_url; - } - issue_request (QUrl (redirect_target.toString ())); - } else { - mp_reply = reply; - m_ready (); - } -} - -void -InputHttpStream::issue_request (const QUrl &url) -{ - delete mp_buffer; - mp_buffer = 0; - - // reset the retry counters -> this way we can detect authentication failures - s_auth_handler->reset (); - - QNetworkRequest request (url); - if (tl::verbosity() >= 30) { - tl::info << "HTTP request URL: " << url.toString ().toUtf8 ().constData (); - } - for (std::map::const_iterator h = m_headers.begin (); h != m_headers.end (); ++h) { - if (tl::verbosity() >= 40) { - tl::info << "HTTP request header: " << h->first << ": " << h->second; - } - request.setRawHeader (QByteArray (h->first.c_str ()), QByteArray (h->second.c_str ())); - } - -#if QT_VERSION < 0x40700 - if (m_request == "GET" && m_data.isEmpty ()) { - mp_active_reply.reset (s_network_manager->get (request)); - } else { - throw tl::Exception (tl::to_string (QObject::tr ("Custom HTTP requests are not supported in this build (verb is %1)").arg (QString::fromUtf8 (m_request)))); - } -#else - if (m_data.isEmpty ()) { - mp_active_reply.reset (s_network_manager->sendCustomRequest (request, m_request)); - } else { - if (tl::verbosity() >= 40) { - tl::info << "HTTP request data: " << m_data.constData (); - } - mp_buffer = new QBuffer (&m_data); - mp_active_reply.reset (s_network_manager->sendCustomRequest (request, m_request, mp_buffer)); - } -#endif -} - -void -InputHttpStream::send () -{ - if (mp_reply == 0) { - issue_request (QUrl (tl::to_qstring (m_url))); - } -} - -size_t -InputHttpStream::read (char *b, size_t n) -{ - // Prevents deferred methods to be executed during the processEvents below (undesired side effects) - tl::NoDeferredMethods silent; - - if (mp_reply == 0) { - issue_request (QUrl (tl::to_qstring (m_url))); - } - - // TODO: progress, timeout - while (mp_reply == 0) { - QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents); - } - - if (mp_reply->error () != QNetworkReply::NoError) { - - // throw an error - std::string em = tl::to_string (mp_reply->attribute (QNetworkRequest::HttpReasonPhraseAttribute).toString ()); - if (tl::verbosity() >= 30) { - tl::info << "HTTP response error: " << em; - } - - int ec = mp_reply->attribute (QNetworkRequest::HttpStatusCodeAttribute).toInt (); - if (ec == 0) { - switch (mp_reply->error ()) { - case QNetworkReply::ConnectionRefusedError: - em = tl::to_string (QObject::tr ("Connection refused")); - break; - case QNetworkReply::RemoteHostClosedError: - em = tl::to_string (QObject::tr ("Remote host closed connection")); - break; - case QNetworkReply::HostNotFoundError: - em = tl::to_string (QObject::tr ("Host not found")); - break; - case QNetworkReply::TimeoutError: - em = tl::to_string (QObject::tr ("Timeout")); - break; - case QNetworkReply::ContentAccessDenied: - em = tl::to_string (QObject::tr ("Access denied")); - break; - case QNetworkReply::ContentNotFoundError: - em = tl::to_string (QObject::tr ("Content not found")); - break; - default: - em = tl::to_string (QObject::tr ("Network API error")); - } - ec = int (mp_reply->error ()); - } - - throw HttpErrorException (em, ec, m_url); - - } - - QByteArray data = mp_reply->read (n); - memcpy (b, data.constData (), data.size ()); - if (tl::verbosity() >= 40) { - tl::info << "HTTP reponse data read: " << data.constData (); - } - return data.size (); -} - -void -InputHttpStream::reset () -{ - throw tl::Exception (tl::to_string (QObject::tr ("'reset' is not supported on HTTP input streams"))); -} - -std::string -InputHttpStream::filename () const -{ - return tl::to_string (QFileInfo (QUrl (tl::to_qstring (m_url)).path ()).fileName ()); -} } diff --git a/src/tl/tl/tlHttpStream.h b/src/tl/tl/tlHttpStream.h index b5c6115d2..b526071af 100644 --- a/src/tl/tl/tlHttpStream.h +++ b/src/tl/tl/tlHttpStream.h @@ -20,26 +20,36 @@ */ - #ifndef HDR_tlHttpStream #define HDR_tlHttpStream -#include "tlStream.h" -#include "tlEvents.h" +#include "tlHttpStreamCurl.h" +#include "tlHttpStreamQt.h" -#include -#include -#include -#include - -class QNetworkAccessManager; -class QNetworkReply; -class QNetworkProxy; -class QAuthenticator; +#include "tlObject.h" namespace tl { +/** + * @brief A callback interface to provide the authentication data + */ +class TL_PUBLIC HttpCredentialProvider + : public tl::Object +{ +public: + HttpCredentialProvider () { } + virtual ~HttpCredentialProvider () { } + + /** + * @brief Gets the user name and password for the given URL and authentication realm + */ + virtual bool user_password (const std::string &url, const std::string &realm, bool proxy, int attempt, std::string &user, std::string &passwd) = 0; +}; + +/** + * @brief An exception class for HTTP errors + */ class TL_PUBLIC HttpErrorException : public tl::Exception { @@ -49,137 +59,6 @@ public: { } }; -class AuthenticationHandler - : public QObject -{ -Q_OBJECT - -public: - AuthenticationHandler (); - -public slots: - void authenticationRequired (QNetworkReply *, QAuthenticator *); - void proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *); - void reset (); - -private: - int m_retry, m_proxy_retry; -}; - -/** - * @brief A http input delegate for tl::InputStream - * - * Implements the reader from a server using the HTTP protocol - */ -class TL_PUBLIC InputHttpStream - : public QObject, public InputStreamBase -{ -Q_OBJECT - -public: - /** - * @brief Open a stream with the given URL - */ - InputHttpStream (const std::string &url); - - /** - * @brief Close the file - * - * The destructor will automatically close the connection. - */ - virtual ~InputHttpStream (); - - /** - * @brief Sends the request for data - * To ensure prompt delivery of data, this method can be used prior to - * "read" to trigger the download from the given URL. - * This method will return immediately. When the reply is available, - * the "ready" event will be triggered. "read" can then be used to - * read the data or - in case of an error - throw an exception. - * If "send" is not used before "read", "read" will block until data - * is available. - * If a request has already been sent, this method will do nothing. - */ - void send (); - - /** - * @brief Sets the request verb - * The default verb is "GET" - */ - void set_request (const char *r); - - /** - * @brief Sets data to be sent with the request - * If data is given, it is sent along with the request. - * This version takes a null-terminated string. - */ - void set_data (const char *data); - - /** - * @brief Sets data to be sent with the request - * If data is given, it is sent along with the request. - * This version takes a data plus length. - */ - void set_data (const char *data, size_t n); - - /** - * @brief Sets a header field - */ - void add_header (const std::string &name, const std::string &value); - - /** - * @brief Read from the stream - * Implements the basic read method. - */ - virtual size_t read (char *b, size_t n); - - /** - * @brief Gets the "ready" event - * Connect to this event for the asynchroneous interface. - */ - tl::Event &ready () - { - return m_ready; - } - - /** - * @brief Gets a value indicating whether data is available - */ - bool data_available () - { - return mp_reply != 0; - } - - virtual void reset (); - - virtual std::string source () const - { - return m_url; - } - - virtual std::string absolute_path () const - { - return m_url; - } - - virtual std::string filename () const; - -private slots: - void finished (QNetworkReply *); - -private: - std::string m_url; - QNetworkReply *mp_reply; - std::auto_ptr mp_active_reply; - QByteArray m_request; - QByteArray m_data; - QBuffer *mp_buffer; - std::map m_headers; - tl::Event m_ready; - - void issue_request (const QUrl &url); -}; - } #endif diff --git a/src/tl/tl/tlHttpStreamCurl.cc b/src/tl/tl/tlHttpStreamCurl.cc new file mode 100644 index 000000000..5d73b222d --- /dev/null +++ b/src/tl/tl/tlHttpStreamCurl.cc @@ -0,0 +1,1176 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2018 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if defined(HAVE_CURL) + +#include "tlHttpStream.h" +#include "tlHttpStreamCurl.h" +#include "tlLog.h" +#include "tlStaticObjects.h" +#include "tlDeferredExecution.h" +#include "tlEvents.h" +#include "tlAssert.h" +#include "tlStaticObjects.h" +#include "tlProgress.h" +#include "tlDeferredExecution.h" + +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +// #define DEBUG_CURL 1 + + +namespace tl +{ + +// --------------------------------------------------------------- +// Utilities + +std::string server_from_url (const std::string &url) +{ + QUrl qurl (tl::to_qstring (url)); + return tl::to_string (qurl.scheme ()) + ":" + tl::to_string (qurl.host ()); +} + +std::string parse_realm (const std::string &header) +{ + std::vector lines = tl::split (header, "\n"); + for (std::vector::const_iterator l = lines.begin (); l != lines.end (); ++l) { + + tl::Extractor ex (l->c_str ()); + std::string header; + if (! ex.try_read_word (header, "_.$-") || ! ex.test (":")) { + continue; + } + + header = tl::to_lower_case (header); + if (header != "www-authenticate" && header != "proxy-authenticate") { + continue; + } + + std::string auth_type; + if (! ex.try_read_word (auth_type)) { + continue; + } + + while (! ex.at_end ()) { + std::string key, value; + if (! ex.try_read_word (key) || ! ex.test ("=") || ! ex.try_read_word_or_quoted (value)) { + break; + } + key = tl::to_lower_case (key); + if (key == "realm") { + return value; + } + } + + } + + return std::string (); +} + + + +// --------------------------------------------------------------- +// CurlCredentialManager definition and implementation + +/** + * @brief A cache for keeping the credentials + */ +class CurlCredentialManager +{ +public: + + enum Mode { + UseAsIs, + Inquire, + ForceInquire + }; + + CurlCredentialManager (bool proxy) + : m_proxy (proxy), mp_provider (0) + { + // .. nothing yet .. + } + + const std::pair *user_password (const std::string &url, const std::string &realm, int attempt, Mode mode) + { + std::string server = server_from_url (url); + + if (mode != ForceInquire) { + + std::map, std::pair >::const_iterator c = m_credentials.find (std::make_pair (server, realm)); + if (c != m_credentials.end ()) { + return &c->second; + } + + } + + if (mode != UseAsIs && mp_provider.get ()) { + + std::string user, password; + if (! mp_provider->user_password (url, realm, m_proxy, attempt, user, password)) { + throw tl::CancelException (); + } + + set_credentials (server, realm, user, password); + return user_password (url, realm, attempt, CurlCredentialManager::UseAsIs); + + } else { + return 0; + } + } + + void set_credentials (const std::string &url, const std::string &realm, const std::string &user, const std::string &passwd) + { + m_credentials[std::make_pair (url, realm)] = std::make_pair (user, passwd); + } + + void set_provider (HttpCredentialProvider *provider) + { + mp_provider.reset (provider); + } + + HttpCredentialProvider *provider () + { + return mp_provider.get (); + } + +private: + std::map, std::pair > m_credentials; + bool m_proxy; + tl::weak_ptr mp_provider; +}; + +// --------------------------------------------------------------- +// ChunkedBuffer definition and implementation + +/** + * @brief A stream buffer + * The stream buffer can push data in chunks and + * deliver data in chunks of different size. + * Internally, the data is kept in the original chunks + * and is combined on output. + */ +class ChunkedBuffer +{ +private: + struct ChunkInfo + { + ChunkInfo () + : pos (0), start (0), size (0) + { + // .. nothing yet .. + } + + ChunkInfo (const ChunkInfo &other) + : pos (0), start (0), size (0) + { + operator= (other); + } + + ChunkInfo &operator= (const ChunkInfo &other) + { + if (this != &other) { + set (other.start, other.size); + pos = start + (other.pos - other.start); + } + return *this; + } + + ~ChunkInfo () + { + set (0, 0); + } + + void set (const char *data, size_t n) + { + if (start) { + delete [] start; + start = pos = 0; + } + + size = n; + if (n > 0) { + char *data_copy = new char [n]; + memcpy (data_copy, data, n); + pos = start = data_copy; + } + } + + size_t fetch (char *data, size_t bytes) + { + size_t n = std::min (bytes, available ()); + if (n > 0) { + memcpy (data, pos, n); + pos += n; + } + return n; + } + + bool empty () const + { + return available () == 0; + } + + size_t available () const + { + return size - (pos - start); + } + + char *pos, *start; + size_t size; + }; + +public: + ChunkedBuffer () + { + // .. nothing yet .. + } + + void clear () + { + m_chunks.clear (); + } + + void push (const char *data, size_t bytes) + { + if (bytes > 0) { + m_chunks.push_back (ChunkInfo ()); + m_chunks.back ().set (data, bytes); + } + } + + size_t fetch (char *data, size_t bytes) + { + char *start = data; + + while (bytes > 0 && ! m_chunks.empty ()) { + + size_t n = m_chunks.front ().fetch (data, bytes); + data += n; + bytes -= n; + if (m_chunks.front ().empty ()) { + m_chunks.pop_front (); + } + + } + + return data - start; + } + + std::string to_string () const + { + std::string s; + s.reserve (size ()); + for (std::list::const_iterator c = m_chunks.begin (); c != m_chunks.end (); ++c) { + s += std::string (c->pos, c->available ()); + } + return s; + } + + size_t size () const + { + size_t bytes = 0; + for (std::list::const_iterator c = m_chunks.begin (); c != m_chunks.end (); ++c) { + bytes += c->size; + } + return bytes; + } + + bool empty () const + { + for (std::list::const_iterator c = m_chunks.begin (); c != m_chunks.end (); ++c) { + if (! c->empty ()) { + return false; + } + } + return true; + } + + std::list m_chunks; +}; + +// --------------------------------------------------------------- +// CurlConnection definition + +/** + * @brief Represents a connection to a server + * + * Objects of this class are created by the CurlNetworkManager and must + * be deleted by the caller when the connection is no longer required. + * + * @code + * CurlNetworkManager mgr; + * std::auto_ptr conn (mgr.create_connection ()); + * conn->set_url ("http://www.example.com"); + * conn->send (); + * while (mgr.tick () > 0) { + * // continue; + * } + * @endcode + */ +class CurlConnection +{ +public: + /** + * @brief Destructor + */ + ~CurlConnection (); + + /** + * @brief Sets the URL to use for the request + * Use "send" to initiate the request. + */ + void set_url (const char *url); + + /** + * @brief Gets the URL + */ + const std::string &url () const + { + return m_url; + } + + /** + * @brief Sets the custom request to use + * This will override the "GET" request used normally. + */ + void set_request (const char *request); + + /** + * @brief Sets a custom header field to use for the request + */ + void add_header (const char *header, const char *value); + + /** + * @brief Sets the request data (0 terminated string) + */ + void set_data (const char *data); + + /** + * @brief Sets the request data + */ + void set_data (const char *data, size_t n); + + /** + * @brief Gets the byte count of the data block read + */ + size_t read_available () const; + + /** + * @brief Fetches a block of read data + */ + size_t fetch_read_data (char *buffer, size_t nbytes); + + /** + * @brief Gets the read data as a string + */ + std::string read_data_to_string () const; + + /** + * @brief Sends the request + * Note: the connection object that sends the request will receive the data. + */ + void send (); + + /** + * @brief Gets the HTTP status after the finished_event has been triggered + */ + int http_status () const + { + return m_http_status; + } + + /** + * @brief Gets a value indicating whether the request has finished + */ + bool finished () const + { + return m_finished; + } + + /** + * @brief Checks the response once finished is true + * This method will throw an exception if an error occured + */ + void check () const; + + /** + * @brief This event is triggered when the transmission has finished + */ + tl::Event finished_event; + + /** + * @brief This event is triggered when new data is available + */ + tl::Event data_available_event; + +private: + // no copying + CurlConnection (const CurlConnection &); + CurlConnection &operator= (const CurlConnection &); + + friend class CurlNetworkManager; + friend size_t read_func (char *buffer, size_t size, size_t nitems, void *userdata); + friend size_t write_func (char *ptr, size_t size, size_t nmemb, void *userdata); + friend size_t write_header_func (char *ptr, size_t size, size_t nmemb, void *userdata); + + void add_read_data (const char *data, size_t n); + void add_header_data (const char *data, size_t n); + size_t fetch_data (char *buffer, size_t nbytes); + + void finished (int status); + void init (); + + CurlConnection (CURL *handle); + + CURL *mp_handle; + ChunkedBuffer m_data, m_read_data, m_header_data; + char m_error_msg [CURL_ERROR_SIZE]; + std::string m_url, m_request; + int m_authenticated; + std::string m_user, m_password; + struct curl_slist *mp_headers; + int m_http_status; + bool m_finished; + int m_status; +}; + +// --------------------------------------------------------------- +// CurlNetworkManager definition + +/** + * @brief A singleton instance to manage the connections + */ +class CurlNetworkManager +{ +public: + /** + * @brief Constructor + * There must be one instance, not more. + */ + CurlNetworkManager (); + + /** + * @brief Destructor + */ + ~CurlNetworkManager (); + + /** + * @brief Creates a connection object + * Connection objects are the basic keys for implementing connections via curl. + * Use the returned object to send and receive data. + * The returned object must be deleted by the caller. + */ + CurlConnection *create_connection (); + + /** + * @brief The credentials manager object + */ + CurlCredentialManager &credentials () + { + return m_credentials; + } + + /** + * @brief The proxy_credentials manager object + */ + CurlCredentialManager &proxy_credentials () + { + return m_proxy_credentials; + } + + /** + * @brief Must be called in regular intervals to update the status + * Returns the number of open connections. + */ + int tick (); + + /** + * @brief The singleton instance + */ + static CurlNetworkManager *instance (); + +private: + // TODO: mutex for thread locking + friend class CurlConnection; + + void add_connection (CurlConnection *connection); + void release_connection (CurlConnection *connection); + void start (CurlConnection *connection); + void on_tick (); + + tl::DeferredMethod dm_tick; + CURLM *mp_multi_handle; + int m_still_running; + std::map m_handle_refcount; + std::map m_handle2connection; + CurlCredentialManager m_credentials; + CurlCredentialManager m_proxy_credentials; + static CurlNetworkManager *ms_instance; +}; + +// ---------------------------------------------------------------------- +// CurlConnection implementation + +size_t read_func (char *buffer, size_t size, size_t nitems, void *userdata); +size_t write_func (char *ptr, size_t size, size_t nmemb, void *userdata); +size_t write_header_func (char *ptr, size_t size, size_t nmemb, void *userdata); + +CurlConnection::CurlConnection (CURL *handle) + : mp_handle (handle) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection(" << (void *)handle << ")" << std::endl; +#endif + init (); +} + +CurlConnection::CurlConnection (const CurlConnection &other) + : mp_handle (other.mp_handle) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection(" << (void *)mp_handle << ")" << std::endl; +#endif + init (); +} + +void CurlConnection::init () +{ + m_http_status = 0; + m_finished = false; + m_status = 0; + mp_headers = 0; + m_authenticated = 0; + + CurlNetworkManager::instance ()->add_connection (this); +} + +CurlConnection::~CurlConnection () +{ +#if defined(DEBUG_CURL) + std::cerr << "~CurlConnection(" << (void *)mp_handle << ")" << std::endl; +#endif + CurlNetworkManager::instance ()->release_connection (this); + curl_slist_free_all (mp_headers); +} + +void CurlConnection::set_url (const char *url) +{ + m_url = url; +} + +void CurlConnection::set_request (const char *request) +{ + m_request = request; +} + +void CurlConnection::add_header (const char *header, const char *value) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection::set_header(" << header << ", " << value << ")" << std::endl; +#endif + if (! value) { + mp_headers = curl_slist_append (mp_headers, (std::string (header) + ";").c_str ()); + } else { + mp_headers = curl_slist_append (mp_headers, (std::string (header) + ": " + std::string (value)).c_str ()); + } +} + +void CurlConnection::set_data (const char *data) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection::set_data(...)" << std::endl; +#endif + m_data.push (data, strlen (data)); +} + +void CurlConnection::set_data (const char *data, size_t n) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection::set_data(" << n << " bytes)" << std::endl; +#endif + m_data.push (data, n); +} + +size_t CurlConnection::read_available () const +{ + return m_read_data.size (); +} + +std::string CurlConnection::read_data_to_string () const +{ + return m_read_data.to_string (); +} + +size_t CurlConnection::fetch_read_data (char *buffer, size_t nbytes) +{ + return m_read_data.fetch (buffer, nbytes); +} + +size_t CurlConnection::fetch_data (char *buffer, size_t nbytes) +{ + return m_data.fetch (buffer, nbytes); +} + +void CurlConnection::send () +{ + m_http_status = 0; + m_status = 0; + m_finished = false; + m_read_data.clear (); + m_header_data.clear (); + + if (tl::verbosity() >= 30) { + tl::info << "HTTP request URL: " << m_url; + if (tl::verbosity() >= 40) { + curl_slist *hl = mp_headers; + tl::info << "HTTP request header: "; + while (hl) { + tl::info << " " << hl->data; + hl = hl->next; + } + tl::info << "HTTP request data: " << m_data.to_string (); + } + } + + curl_easy_setopt (mp_handle, CURLOPT_URL, m_url.c_str ()); + if (! m_request.empty ()) { + curl_easy_setopt (mp_handle, CURLOPT_CUSTOMREQUEST, m_request.c_str ()); + } + +#if defined(DEBUG_CURL) + curl_easy_setopt (mp_handle, CURLOPT_VERBOSE, 1L); +#endif + + curl_easy_setopt (mp_handle, CURLOPT_ERRORBUFFER, &m_error_msg); + + curl_easy_setopt (mp_handle, CURLOPT_READFUNCTION, &read_func); + curl_easy_setopt (mp_handle, CURLOPT_READDATA, (void *) this); + curl_easy_setopt (mp_handle, CURLOPT_WRITEFUNCTION, &write_func); + curl_easy_setopt (mp_handle, CURLOPT_WRITEDATA, (void *) this); + curl_easy_setopt (mp_handle, CURLOPT_HEADERFUNCTION, &write_header_func); + curl_easy_setopt (mp_handle, CURLOPT_HEADERDATA, (void *) this); + + if (! m_data.empty ()) { + curl_easy_setopt (mp_handle, CURLOPT_UPLOAD, 1L); + curl_easy_setopt (mp_handle, CURLOPT_INFILESIZE, (long) m_data.size ()); + } else { + curl_easy_setopt (mp_handle, CURLOPT_UPLOAD, 0L); + } + + curl_easy_setopt (mp_handle, CURLOPT_HTTPHEADER, mp_headers); + + if (m_authenticated > 0) { + curl_easy_setopt (mp_handle, CURLOPT_PASSWORD, m_password.c_str ()); + curl_easy_setopt (mp_handle, CURLOPT_USERNAME, m_user.c_str ()); + } + + // always resolve redirects + curl_easy_setopt (mp_handle, CURLOPT_FOLLOWLOCATION, 1L); + + CurlNetworkManager::instance ()->start (this); +} + +void CurlConnection::add_read_data (const char *data, size_t n) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection::add_read_data(" << n << " bytes)" << std::endl; +#endif + m_read_data.push (data, n); + data_available_event (); +} + +void CurlConnection::add_header_data (const char *data, size_t n) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection::add_header_data(" << n << " bytes)" << std::endl; +#endif + m_header_data.push (data, n); +} + +void CurlConnection::check () const +{ + if (m_status < 0) { + + throw tl::CancelException (); + + } else if (m_status != 0) { + + throw tl::HttpErrorException (tl::to_string (QObject::tr ("Connection error (%1)").arg (QString::fromLatin1 (m_error_msg))), m_status, m_url); + + } else if (m_http_status < 200 || m_http_status >= 300) { + + // translate some known errors + const char *error_text = 0; + if (m_http_status == 400) { + error_text = "Bad Request"; + } else if (m_http_status == 401) { + error_text = "Unauthorized"; + } else if (m_http_status == 403) { + error_text = "Forbidden"; + } else if (m_http_status == 404) { + error_text = "Not Found"; + } else if (m_http_status == 405) { + error_text = "Method Not Allowed"; + } else if (m_http_status == 406) { + error_text = "Not Acceptable"; + } else if (m_http_status == 407) { + error_text = "Proxy Authentication Required"; + } else if (m_http_status == 408) { + error_text = "Request Timeout"; + } + if (error_text) { + throw tl::HttpErrorException (error_text, m_http_status, m_url); + } else { + throw tl::HttpErrorException (tl::to_string (QObject::tr ("HTTP error")), m_http_status, m_url); + } + + } +} + +void CurlConnection::finished (int status) +{ + if (status != 0) { + m_status = status; + m_finished = true; + finished_event (); + return; + } + + long http_code = -1; + curl_easy_getinfo (mp_handle, CURLINFO_RESPONSE_CODE, &http_code); + + if (tl::verbosity() >= 30) { + tl::info << "HTTP response code: " << http_code; + if (tl::verbosity() >= 40) { + tl::info << "HTTP response header: " << m_header_data.to_string (); + } + } + + if ((http_code == 401 || http_code == 407)) { + + bool proxy_auth = (http_code == 407); + +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection::authentication required" << std::endl; +#endif + + // Authentication required: analyse header for realm information + std::string realm = parse_realm (m_header_data.to_string ()); + + const std::pair *pwd; + try { + + // Note: on the second attempt use ForceInquire, so we don't reuse the wrong credentials + CurlCredentialManager::Mode mode = m_authenticated == 0 ? CurlCredentialManager::Inquire : CurlCredentialManager::ForceInquire; + + if (proxy_auth) { + pwd = CurlNetworkManager::instance ()->proxy_credentials ().user_password (m_url, realm, m_authenticated + 1, mode); + } else { + pwd = CurlNetworkManager::instance ()->credentials ().user_password (m_url, realm, m_authenticated + 1, mode); + } + + } catch (tl::CancelException &) { + + m_finished = true; + m_status = -1; // cancelled + finished_event (); + return; + + } + + if (pwd) { + + // restart - this time with authentication + + m_user = pwd->first; + m_password = pwd->second; + m_authenticated += 1; + +#if defined(DEBUG_CURL) + std::cerr << "CurlConnection::finished: resend with authentication data" << std::endl; +#endif + + curl_easy_reset (mp_handle); + + send (); + return; + + } + + } + + m_http_status = http_code; + m_finished = true; + finished_event (); +} + +size_t read_func(char *buffer, size_t size, size_t nitems, void *userdata) +{ + CurlConnection *connection = (CurlConnection *) userdata; + return connection->fetch_data (buffer, size * nitems); +} + +size_t write_func(char *buffer, size_t size, size_t nitems, void *userdata) +{ + CurlConnection *connection = (CurlConnection *) userdata; + connection->add_read_data (buffer, size * nitems); + return size * nitems; +} + +size_t write_header_func(char *buffer, size_t size, size_t nitems, void *userdata) +{ + CurlConnection *connection = (CurlConnection *) userdata; + connection->add_header_data (buffer, size * nitems); + return size * nitems; +} + + +// ---------------------------------------------------------------------- +// CurlNetworkManager implementation + +CurlNetworkManager *CurlNetworkManager::ms_instance = 0; + +CurlNetworkManager::CurlNetworkManager () + : dm_tick (this, &CurlNetworkManager::on_tick), + m_still_running (0), m_credentials (false), m_proxy_credentials (true) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlNetworkManager()" << std::endl; +#endif + tl_assert (ms_instance == 0); + mp_multi_handle = curl_multi_init (); + ms_instance = this; + + // will register the object in the static object repo - it's cleaned up properly + // when the application terminates. + tl::StaticObjects::reg (&ms_instance); +} + +CurlNetworkManager::~CurlNetworkManager () +{ +#if defined(DEBUG_CURL) + std::cerr << "~CurlNetworkManager()" << std::endl; +#endif + if (ms_instance == this) { + ms_instance = 0; + } + curl_multi_cleanup (mp_multi_handle); +} + +CurlNetworkManager *CurlNetworkManager::instance () +{ + if (! ms_instance) { + new CurlNetworkManager (); + } + return ms_instance; +} + +CurlConnection *CurlNetworkManager::create_connection () +{ + CURL *handle = curl_easy_init(); + return new CurlConnection (handle); +} + +void CurlNetworkManager::start (CurlConnection *connection) +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlNetworkManager::start(" << (void*)connection->mp_handle << ")" << std::endl; +#endif + curl_multi_add_handle (mp_multi_handle, connection->mp_handle); + curl_multi_perform(mp_multi_handle, &m_still_running); + m_handle2connection[connection->mp_handle] = connection; + + dm_tick (); +} + +void CurlNetworkManager::add_connection (CurlConnection *connection) +{ + ++m_handle_refcount[connection->mp_handle]; +} + +void CurlNetworkManager::release_connection (CurlConnection *connection) +{ + --m_handle_refcount[connection->mp_handle]; + if (m_handle_refcount[connection->mp_handle] == 0) { + +#if defined(DEBUG_CURL) + std::cerr << "CurlNetworkManager::release_connection(" << (void*)connection->mp_handle << ")" << std::endl; +#endif + curl_easy_cleanup(connection->mp_handle); + m_handle_refcount.erase (connection->mp_handle); + + std::map::iterator h = m_handle2connection.find (connection->mp_handle); + if (h != m_handle2connection.end ()) { + m_handle2connection.erase (h); + } + + } +} + +void CurlNetworkManager::on_tick () +{ + if (tick ()) { + dm_tick (); + } +} + +int CurlNetworkManager::tick () +{ +#if defined(DEBUG_CURL) + std::cerr << "CurlNetworkManager::tick()" << std::endl; +#endif + if (m_still_running <= 0) { + return 0; + } + + struct timeval timeout; + int rc; // select() return code + CURLMcode mc; // curl_multi_fdset() return code + + fd_set fdread; + fd_set fdwrite; + fd_set fdexcep; + int maxfd = -1; + + long curl_timeo = -1; + + FD_ZERO (&fdread); + FD_ZERO (&fdwrite); + FD_ZERO (&fdexcep); + + // set a suitable timeout to play around with + timeout.tv_sec = 1; + timeout.tv_usec = 0; + + curl_multi_timeout (mp_multi_handle, &curl_timeo); + if (curl_timeo >= 0) { + timeout.tv_sec = curl_timeo / 1000; + if (timeout.tv_sec > 1) { + timeout.tv_sec = 1; + } else { + timeout.tv_usec = (curl_timeo % 1000) * 1000; + } + } + + // get file descriptors from the transfers + mc = curl_multi_fdset (mp_multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); + + if (mc != CURLM_OK) { + throw tl::HttpErrorException (tl::to_string (QObject::tr ("Connection error (curl_multi_fdset() failed)")), mc, std::string ()); + } + + /* On success the value of maxfd is guaranteed to be >= -1. We call + select(maxfd + 1, ...); specially in case of (maxfd == -1) there are + no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- + to sleep 100ms, which is the minimum suggested value in the + curl_multi_fdset() doc. */ + if (maxfd == -1) { +#ifdef _WIN32 + Sleep (100); + rc = 0; +#else + // Portable sleep for platforms other than Windows. + struct timeval wait = { 0, 10 * 1000 }; /* 10ms */ + rc = select (0, NULL, NULL, NULL, &wait); +#endif + } + else { + /* Note that on some platforms 'timeout' may be modified by select(). + If you need access to the original value save a copy beforehand. */ + rc = select (maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); + } + + int sr = m_still_running; + + switch (rc) { + case -1: + // select error + break; + case 0: // timeout + default: // action + curl_multi_perform (mp_multi_handle, &m_still_running); + break; + } + + if (m_still_running < sr) { + + // less running connections: analyse and finish + CURLMsg *msg; + int msgs_left; + + while ((msg = curl_multi_info_read (mp_multi_handle, &msgs_left))) { + + if (msg->msg != CURLMSG_DONE) { + continue; + } + + std::map::iterator h = m_handle2connection.find (msg->easy_handle); + if (h != m_handle2connection.end ()) { + curl_multi_remove_handle (mp_multi_handle, msg->easy_handle); + h->second->finished (msg->data.result); + } + + } + + } + + return m_still_running; +} + +// --------------------------------------------------------------- +// InputHttpFileCurl implementation + +InputHttpStream::InputHttpStream (const std::string &url) +{ + m_sent = false; + + m_connection.reset (CurlNetworkManager::instance ()->create_connection ()); + m_connection->set_url (url.c_str ()); + m_connection->finished_event.add (this, &InputHttpStream::finished); +} + +InputHttpStream::~InputHttpStream () +{ + // .. nothing yet .. +} + +bool +InputHttpStream::data_available () +{ + return m_connection->read_available () > 0; +} + +void +InputHttpStream::set_request (const char *r) +{ + m_connection->set_request (r); +} + +void +InputHttpStream::set_data (const char *data) +{ + m_connection->set_data (data); +} + +void +InputHttpStream::set_data (const char *data, size_t n) +{ + m_connection->set_data (data, n); +} + +void +InputHttpStream::add_header (const std::string &name, const std::string &value) +{ + m_connection->add_header (name.c_str (), value.c_str ()); +} + +void +InputHttpStream::finished () +{ + m_ready_event (); +} + +void +InputHttpStream::send () +{ + m_connection->send (); + m_sent = true; +} + +size_t +InputHttpStream::read (char *b, size_t n) +{ + if (! m_sent) { + + send (); + + // Prevents deferred methods to be executed during the processEvents below (undesired side effects) + tl::NoDeferredMethods silent; + + // TODO: progress, timeout + tl::AbsoluteProgress progress (tl::to_string (QObject::tr ("Downloading")) + " " + m_connection->url (), 1); + while (! m_connection->finished () && CurlNetworkManager::instance ()->tick ()) { + ++progress; + } + + tl_assert (m_connection->finished ()); + m_connection->check (); + + if (tl::verbosity() >= 40) { + tl::info << "HTTP reponse data read: " << m_connection->read_data_to_string (); + } + + } + + return m_connection->fetch_read_data (b, n); +} + +void +InputHttpStream::reset () +{ + throw tl::Exception (tl::to_string (QObject::tr ("'reset' is not supported on HTTP input streams"))); +} + +std::string +InputHttpStream::filename () const +{ + return tl::to_string (QFileInfo (QUrl (tl::to_qstring (m_connection->url ())).path ()).fileName ()); +} + +std::string +InputHttpStream::source () const +{ + return m_connection->url (); +} + +std::string +InputHttpStream::absolute_path () const +{ + return m_connection->url (); +} + +void +InputHttpStream::set_credential_provider (HttpCredentialProvider *cp) +{ + CurlNetworkManager::instance ()->credentials ().set_provider (cp); + CurlNetworkManager::instance ()->proxy_credentials ().set_provider (cp); +} + +} + +#endif diff --git a/src/tl/tl/tlHttpStreamCurl.h b/src/tl/tl/tlHttpStreamCurl.h new file mode 100644 index 000000000..4d7abf3d0 --- /dev/null +++ b/src/tl/tl/tlHttpStreamCurl.h @@ -0,0 +1,144 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2018 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef HDR_tlHttpStreamCurl +#define HDR_tlHttpStreamCurl + +#if defined(HAVE_CURL) + +#include "tlStream.h" +#include "tlEvents.h" +#include "tlObject.h" + +#include + +namespace tl +{ + +class CurlConnection; +class HttpCredentialProvider; + +/** + * @brief A http input delegate for tl::InputStream + * + * Implements the reader from a server using the HTTP protocol + */ +class TL_PUBLIC InputHttpStream + // NOTE: QObject is required because we use "deleteLater" + : public QObject, public tl::Object, public InputStreamBase +{ +public: + /** + * @brief Open a stream with the given URL + */ + InputHttpStream (const std::string &url); + + /** + * @brief Close the file + * + * The destructor will automatically close the connection. + */ + virtual ~InputHttpStream (); + + /** + * @brief Sets the credential provider + */ + static void set_credential_provider (HttpCredentialProvider *cp); + + /** + * @brief Sends the request for data + * To ensure prompt delivery of data, this method can be used prior to + * "read" to trigger the download from the given URL. + * This method will return immediately. When the reply is available, + * the "ready" event will be triggered. "read" can then be used to + * read the data or - in case of an error - throw an exception. + * If "send" is not used before "read", "read" will block until data + * is available. + * If a request has already been sent, this method will do nothing. + */ + void send (); + + /** + * @brief Sets the request verb + * The default verb is "GET" + */ + void set_request (const char *r); + + /** + * @brief Sets data to be sent with the request + * If data is given, it is sent along with the request. + * This version takes a null-terminated string. + */ + void set_data (const char *data); + + /** + * @brief Sets data to be sent with the request + * If data is given, it is sent along with the request. + * This version takes a data plus length. + */ + void set_data (const char *data, size_t n); + + /** + * @brief Sets a header field + */ + void add_header (const std::string &name, const std::string &value); + + /** + * @brief Read from the stream + * Implements the basic read method. + */ + virtual size_t read (char *b, size_t n); + + /** + * @brief Gets the "ready" event + * Connect to this event for the asynchroneous interface. + */ + tl::Event &ready () + { + return m_ready_event; + } + + /** + * @brief Gets a value indicating whether data is available + */ + bool data_available (); + + virtual void reset (); + + virtual std::string source () const; + virtual std::string absolute_path () const; + virtual std::string filename () const; + +private: + std::auto_ptr m_connection; + tl::Event m_ready_event; + bool m_sent; + + void finished (); +}; + +} + +#endif + +#endif + diff --git a/src/tl/tl/tlHttpStreamQt.cc b/src/tl/tl/tlHttpStreamQt.cc new file mode 100644 index 000000000..50e26e3fe --- /dev/null +++ b/src/tl/tl/tlHttpStreamQt.cc @@ -0,0 +1,296 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2018 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if !defined(HAVE_CURL) + +#include "tlHttpStream.h" +#include "tlLog.h" +#include "tlStaticObjects.h" +#include "tlDeferredExecution.h" +#include "tlObject.h" + +#include "ui_PasswordDialog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace tl +{ + +tl::weak_ptr sp_credential_provider; + +// --------------------------------------------------------------- +// AuthenticationHandler implementation + +AuthenticationHandler::AuthenticationHandler () + : QObject (0), m_retry (0), m_proxy_retry (0) +{ + // .. nothing yet .. +} + +void +AuthenticationHandler::reset () +{ + m_retry = 0; + m_proxy_retry = 0; +} + +void +AuthenticationHandler::authenticationRequired (QNetworkReply *reply, QAuthenticator *auth) +{ + if (sp_credential_provider.get ()) { + + // TODO: how to cancel? + std::string user, passwd; + if (sp_credential_provider->user_password (tl::to_string (reply->url ().toString ()), tl::to_string (auth->realm ()), true, ++m_retry, user, passwd)) { + auth->setPassword (tl::to_qstring (passwd)); + auth->setUser (tl::to_qstring (user)); + } + + } +} + +void +AuthenticationHandler::proxyAuthenticationRequired (const QNetworkProxy &proxy, QAuthenticator *auth) +{ + if (sp_credential_provider.get ()) { + + // TODO: how to cancel? + std::string user, passwd; + if (sp_credential_provider->user_password (tl::to_string (proxy.hostName ()), tl::to_string (auth->realm ()), true, ++m_proxy_retry, user, passwd)) { + auth->setPassword (tl::to_qstring (passwd)); + auth->setUser (tl::to_qstring (user)); + } + + } +} + +// --------------------------------------------------------------- +// InputHttpFileQt implementation + +static QNetworkAccessManager *s_network_manager (0); +static AuthenticationHandler *s_auth_handler (0); + +InputHttpStream::InputHttpStream (const std::string &url) + : m_url (url), mp_reply (0), m_request ("GET"), mp_buffer (0) +{ + if (! s_network_manager) { + + s_network_manager = new QNetworkAccessManager (0); + s_auth_handler = new AuthenticationHandler (); + connect (s_network_manager, SIGNAL (authenticationRequired (QNetworkReply *, QAuthenticator *)), s_auth_handler, SLOT (authenticationRequired (QNetworkReply *, QAuthenticator *))); + connect (s_network_manager, SIGNAL (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *)), s_auth_handler, SLOT (proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *))); + + tl::StaticObjects::reg (&s_network_manager); + tl::StaticObjects::reg (&s_auth_handler); + + } + + connect (s_network_manager, SIGNAL (finished (QNetworkReply *)), this, SLOT (finished (QNetworkReply *))); +} + +InputHttpStream::~InputHttpStream () +{ + // .. nothing yet .. +} + +void +InputHttpStream::set_credential_provider (HttpCredentialProvider *cp) +{ + sp_credential_provider.reset (cp); +} + +void +InputHttpStream::set_request (const char *r) +{ + m_request = QByteArray (r); +} + +void +InputHttpStream::set_data (const char *data) +{ + m_data = QByteArray (data); +} + +void +InputHttpStream::set_data (const char *data, size_t n) +{ + m_data = QByteArray (data, int (n)); +} + +void +InputHttpStream::add_header (const std::string &name, const std::string &value) +{ + m_headers.insert (std::make_pair (name, value)); +} + +void +InputHttpStream::finished (QNetworkReply *reply) +{ + if (reply != mp_active_reply.get ()) { + return; + } + + QVariant redirect_target = reply->attribute (QNetworkRequest::RedirectionTargetAttribute); + if (reply->error () == QNetworkReply::NoError && ! redirect_target.isNull ()) { + m_url = tl::to_string (redirect_target.toString ()); + if (tl::verbosity() >= 30) { + tl::info << "HTTP redirect to: " << m_url; + } + issue_request (QUrl (redirect_target.toString ())); + } else { + mp_reply = reply; + m_ready (); + } +} + +void +InputHttpStream::issue_request (const QUrl &url) +{ + delete mp_buffer; + mp_buffer = 0; + + // reset the retry counters -> this way we can detect authentication failures + s_auth_handler->reset (); + + QNetworkRequest request (url); + if (tl::verbosity() >= 30) { + tl::info << "HTTP request URL: " << url.toString ().toUtf8 ().constData (); + } + for (std::map::const_iterator h = m_headers.begin (); h != m_headers.end (); ++h) { + if (tl::verbosity() >= 40) { + tl::info << "HTTP request header: " << h->first << ": " << h->second; + } + request.setRawHeader (QByteArray (h->first.c_str ()), QByteArray (h->second.c_str ())); + } + +#if QT_VERSION < 0x40700 + if (m_request == "GET" && m_data.isEmpty ()) { + mp_active_reply.reset (s_network_manager->get (request)); + } else { + throw tl::Exception (tl::to_string (QObject::tr ("Custom HTTP requests are not supported in this build (verb is %1)").arg (QString::fromUtf8 (m_request)))); + } +#else + if (m_data.isEmpty ()) { + mp_active_reply.reset (s_network_manager->sendCustomRequest (request, m_request)); + } else { + if (tl::verbosity() >= 40) { + tl::info << "HTTP request data: " << m_data.constData (); + } + mp_buffer = new QBuffer (&m_data); + mp_active_reply.reset (s_network_manager->sendCustomRequest (request, m_request, mp_buffer)); + } +#endif +} + +void +InputHttpStream::send () +{ + if (mp_reply == 0) { + issue_request (QUrl (tl::to_qstring (m_url))); + } +} + +size_t +InputHttpStream::read (char *b, size_t n) +{ + // Prevents deferred methods to be executed during the processEvents below (undesired side effects) + tl::NoDeferredMethods silent; + + if (mp_reply == 0) { + issue_request (QUrl (tl::to_qstring (m_url))); + } + + // TODO: progress, timeout + while (mp_reply == 0) { + QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents); + } + + if (mp_reply->error () != QNetworkReply::NoError) { + + // throw an error + std::string em = tl::to_string (mp_reply->attribute (QNetworkRequest::HttpReasonPhraseAttribute).toString ()); + if (tl::verbosity() >= 30) { + tl::info << "HTTP response error: " << em; + } + + int ec = mp_reply->attribute (QNetworkRequest::HttpStatusCodeAttribute).toInt (); + if (ec == 0) { + switch (mp_reply->error ()) { + case QNetworkReply::ConnectionRefusedError: + em = tl::to_string (QObject::tr ("Connection refused")); + break; + case QNetworkReply::RemoteHostClosedError: + em = tl::to_string (QObject::tr ("Remote host closed connection")); + break; + case QNetworkReply::HostNotFoundError: + em = tl::to_string (QObject::tr ("Host not found")); + break; + case QNetworkReply::TimeoutError: + em = tl::to_string (QObject::tr ("Timeout")); + break; + case QNetworkReply::ContentAccessDenied: + em = tl::to_string (QObject::tr ("Access denied")); + break; + case QNetworkReply::ContentNotFoundError: + em = tl::to_string (QObject::tr ("Content not found")); + break; + default: + em = tl::to_string (QObject::tr ("Network API error")); + } + ec = int (mp_reply->error ()); + } + + throw HttpErrorException (em, ec, m_url); + + } + + QByteArray data = mp_reply->read (n); + memcpy (b, data.constData (), data.size ()); + if (tl::verbosity() >= 40) { + tl::info << "HTTP reponse data read: " << data.constData (); + } + return data.size (); +} + +void +InputHttpStream::reset () +{ + throw tl::Exception (tl::to_string (QObject::tr ("'reset' is not supported on HTTP input streams"))); +} + +std::string +InputHttpStream::filename () const +{ + return tl::to_string (QFileInfo (QUrl (tl::to_qstring (m_url)).path ()).fileName ()); +} + +} + +#endif diff --git a/src/tl/tl/tlHttpStreamQt.h b/src/tl/tl/tlHttpStreamQt.h new file mode 100644 index 000000000..f2c2b8e62 --- /dev/null +++ b/src/tl/tl/tlHttpStreamQt.h @@ -0,0 +1,188 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2018 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlHttpStreamQt +#define HDR_tlHttpStreamQt + +#if !defined(HAVE_CURL) + +#include "tlStream.h" +#include "tlEvents.h" + +#include +#include +#include +#include + +class QNetworkAccessManager; +class QNetworkReply; +class QNetworkProxy; +class QAuthenticator; + +namespace tl +{ + +class HttpCredentialProvider; + +class AuthenticationHandler + : public QObject +{ +Q_OBJECT + +public: + AuthenticationHandler (); + +public slots: + void authenticationRequired (QNetworkReply *, QAuthenticator *); + void proxyAuthenticationRequired (const QNetworkProxy &, QAuthenticator *); + void reset (); + +private: + int m_retry, m_proxy_retry; +}; + +/** + * @brief A http input delegate for tl::InputStream + * + * Implements the reader from a server using the HTTP protocol + */ +class TL_PUBLIC InputHttpStream + : public QObject, public InputStreamBase +{ +Q_OBJECT + +public: + /** + * @brief Open a stream with the given URL + */ + InputHttpStream (const std::string &url); + + /** + * @brief Close the file + * + * The destructor will automatically close the connection. + */ + virtual ~InputHttpStream (); + + /** + * @brief Sets the credential provider + */ + static void set_credential_provider (HttpCredentialProvider *cp); + + /** + * @brief Sends the request for data + * To ensure prompt delivery of data, this method can be used prior to + * "read" to trigger the download from the given URL. + * This method will return immediately. When the reply is available, + * the "ready" event will be triggered. "read" can then be used to + * read the data or - in case of an error - throw an exception. + * If "send" is not used before "read", "read" will block until data + * is available. + * If a request has already been sent, this method will do nothing. + */ + void send (); + + /** + * @brief Sets the request verb + * The default verb is "GET" + */ + void set_request (const char *r); + + /** + * @brief Sets data to be sent with the request + * If data is given, it is sent along with the request. + * This version takes a null-terminated string. + */ + void set_data (const char *data); + + /** + * @brief Sets data to be sent with the request + * If data is given, it is sent along with the request. + * This version takes a data plus length. + */ + void set_data (const char *data, size_t n); + + /** + * @brief Sets a header field + */ + void add_header (const std::string &name, const std::string &value); + + /** + * @brief Read from the stream + * Implements the basic read method. + */ + virtual size_t read (char *b, size_t n); + + /** + * @brief Gets the "ready" event + * Connect to this event for the asynchroneous interface. + */ + tl::Event &ready () + { + return m_ready; + } + + /** + * @brief Gets a value indicating whether data is available + */ + bool data_available () + { + return mp_reply != 0; + } + + virtual void reset (); + + virtual std::string source () const + { + return m_url; + } + + virtual std::string absolute_path () const + { + return m_url; + } + + virtual std::string filename () const; + +private slots: + void finished (QNetworkReply *); + +private: + std::string m_url; + QNetworkReply *mp_reply; + std::auto_ptr mp_active_reply; + QByteArray m_request; + QByteArray m_data; + QBuffer *mp_buffer; + std::map m_headers; + tl::Event m_ready; + + void issue_request (const QUrl &url); +}; + +} + +#endif + +#endif +