packagechooser: (wip) initial LoaderQueue implement

src/modules/packagechooser/CMakeLists.txt

@@ -49,6 +49,7 @@ calamares_add_plugin(packagechooser
         PackageChooserPage.cpp
         PackageChooserViewStep.cpp
         PackageModel.cpp
+        LoaderQueue.cpp
         ${_extra_src}
     RESOURCES
         packagechooser.qrc

src/modules/packagechooser/Config.cpp

@@ -10,6 +10,8 @@
 
 #include "Config.h"
 
+#include "LoaderQueue.h"
+
 #ifdef HAVE_APPDATA
 #include "ItemAppData.h"
 #endif
@@ -349,6 +351,27 @@ Config::setConfigurationMap( const QVariantMap& configurationMap )
         }
     }
 
+    // Lastly, load the groups data
+    const QString key = QStringLiteral( "groupsUrl" );
+    const auto& groupsUrlVariant = configurationMap.value( key );
+    m_queue = new LoaderQueue( this );
+    if ( Calamares::typeOf( groupsUrlVariant ) == Calamares::StringVariantType )
+    {
+        m_queue->append( SourceItem::makeSourceItem( groupsUrlVariant.toString(), configurationMap ) );
+    }
+    else if ( Calamares::typeOf( groupsUrlVariant ) == Calamares::ListVariantType )
+    {
+        for ( const auto& s : groupsUrlVariant.toStringList() )
+        {
+            m_queue->append( SourceItem::makeSourceItem( s, configurationMap ) );
+        }
+    }
+
+    setStatus( required() ? Status::FailedNoData : Status::Ok );
+    cDebug() << "Loading netinstall from" << m_queue->count() << "alternate sources.";
+    connect( m_queue, &LoaderQueue::done, this, &Config::loadingDone );
+    m_queue->load();
+
     bool labels_ok = false;
     auto labels = Calamares::getSubMap( configurationMap, "labels", labels_ok );
     if ( labels_ok )

src/modules/packagechooser/Config.h

@@ -19,6 +19,8 @@
 #include <memory>
 #include <optional>
 
+class LoaderQueue;
+
 enum class PackageChooserMode
 {
     Optional,  // zero or one
@@ -122,6 +124,8 @@ private:
      */
     std::optional< QString > m_packageChoice;
     Calamares::Locale::TranslatedString* m_stepName;  // As it appears in the sidebar
+
+    LoaderQueue* m_queue = nullptr;
 };
 
 

src/modules/packagechooser/LoaderQueue.cpp

@@ -0,0 +1,205 @@
+/*
+ *   SPDX-FileCopyrightText: 2016 Luca Giambonini <almack@chakraos.org>
+ *   SPDX-FileCopyrightText: 2016 Lisa Vitolo     <shainer@chakraos.org>
+ *   SPDX-FileCopyrightText: 2017 Kyle Robbertze  <krobbertze@gmail.com>
+ *   SPDX-FileCopyrightText: 2017-2018 2020, Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#include "LoaderQueue.h"
+
+#include "Config.h"
+#include "network/Manager.h"
+#include "utils/Logger.h"
+#include "utils/RAII.h"
+#include "utils/Yaml.h"
+
+#include <QNetworkReply>
+#include <QTimer>
+
+/** @brief Call fetchNext() on the queue if it can
+ *
+ * On destruction, a new call to fetchNext() is queued, so that
+ * the queue continues loading. Calling release() before the
+ * destructor skips the fetchNext(), ending the queue-loading.
+ *
+ * Calling done(b) is a conditional release: if @p b is @c true,
+ * queues a call to done() on the queue and releases it; otherwise,
+ * does nothing.
+ */
+class FetchNextUnless
+{
+public:
+    FetchNextUnless( LoaderQueue* q )
+        : m_q( q )
+    {
+    }
+    ~FetchNextUnless()
+    {
+        if ( m_q )
+        {
+            QMetaObject::invokeMethod( m_q, "fetchNext", Qt::QueuedConnection );
+        }
+    }
+    void release() { m_q = nullptr; }
+    void done( bool b )
+    {
+        if ( b )
+        {
+            if ( m_q )
+            {
+                QMetaObject::invokeMethod( m_q, "done", Qt::QueuedConnection );
+            }
+            release();
+        }
+    }
+
+private:
+    LoaderQueue* m_q = nullptr;
+};
+
+SourceItem
+SourceItem::makeSourceItem( const QString& groupsUrl, const QVariantMap& configurationMap )
+{
+    if ( groupsUrl == QStringLiteral( "local" ) )
+    {
+        return SourceItem { QUrl(), configurationMap.value( "groups" ).toList() };
+    }
+    else
+    {
+        return SourceItem { QUrl { groupsUrl }, QVariantList() };
+    }
+}
+
+LoaderQueue::LoaderQueue( Config* parent )
+    : QObject( parent )
+    , m_config( parent )
+{
+}
+
+void
+LoaderQueue::append( SourceItem&& i )
+{
+    m_queue.append( std::move( i ) );
+}
+
+void
+LoaderQueue::load()
+{
+    QMetaObject::invokeMethod( this, "fetchNext", Qt::QueuedConnection );
+}
+
+void
+LoaderQueue::fetchNext()
+{
+    if ( m_queue.isEmpty() )
+    {
+        emit done();
+        return;
+    }
+
+    auto source = m_queue.takeFirst();
+    if ( source.isLocal() )
+    {
+        m_config->loadGroupList( source.data );
+        emit done();
+    }
+    else
+    {
+        fetch( source.url );
+    }
+}
+
+void
+LoaderQueue::fetch( const QUrl& url )
+{
+    FetchNextUnless next( this );
+
+    if ( !url.isValid() )
+    {
+        m_config->setStatus( Config::Status::FailedBadConfiguration );
+        cDebug() << "Invalid URL" << url;
+        return;
+    }
+
+    using namespace Calamares::Network;
+
+    cDebug() << "NetInstall loading groups from" << url;
+    QNetworkReply* reply = Manager().asynchronousGet(
+        url,
+        RequestOptions( RequestOptions::FakeUserAgent | RequestOptions::FollowRedirect, std::chrono::seconds( 30 ) ) );
+
+    if ( !reply )
+    {
+        cDebug() << Logger::SubEntry << "Request failed immediately.";
+        // If nobody sets a different status, this will remain
+        m_config->setStatus( Config::Status::FailedBadConfiguration );
+    }
+    else
+    {
+        // When the network request is done, **then** we might
+        // do the next item from the queue, so don't call fetchNext() now.
+        next.release();
+        m_reply = reply;
+        connect( reply, &QNetworkReply::finished, this, &LoaderQueue::dataArrived );
+    }
+}
+
+void
+LoaderQueue::dataArrived()
+{
+    FetchNextUnless next( this );
+
+    if ( !m_reply || !m_reply->isFinished() )
+    {
+        cWarning() << "NetInstall data called too early.";
+        m_config->setStatus( Config::Status::FailedInternalError );
+        return;
+    }
+
+    cDebug() << "NetInstall group data received" << m_reply->size() << "bytes from" << m_reply->url();
+
+    cqDeleter< QNetworkReply > d { m_reply };
+
+    // If m_required is *false* then we still say we're ready
+    // even if the reply is corrupt or missing.
+    if ( m_reply->error() != QNetworkReply::NoError )
+    {
+        cWarning() << "unable to fetch netinstall package lists.";
+        cDebug() << Logger::SubEntry << "Netinstall reply error: " << m_reply->error();
+        cDebug() << Logger::SubEntry << "Request for url: " << m_reply->url().toString()
+                 << " failed with: " << m_reply->errorString();
+        m_config->setStatus( Config::Status::FailedNetworkError );
+        return;
+    }
+
+    QByteArray yamlData = m_reply->readAll();
+    try
+    {
+        auto groups = ::YAML::Load( yamlData.constData() );
+
+        if ( groups.IsSequence() )
+        {
+            m_config->loadGroupList( Calamares::YAML::sequenceToVariant( groups ) );
+            next.done( m_config->statusCode() == Config::Status::Ok );
+        }
+        else if ( groups.IsMap() )
+        {
+            auto map = Calamares::YAML::mapToVariant( groups );
+            m_config->loadGroupList( map.value( "groups" ).toList() );
+            next.done( m_config->statusCode() == Config::Status::Ok );
+        }
+        else
+        {
+            cWarning() << "NetInstall groups data does not form a sequence.";
+        }
+    }
+    catch ( ::YAML::Exception& e )
+    {
+        Calamares::YAML::explainException( e, yamlData, "netinstall groups data" );
+        m_config->setStatus( Config::Status::FailedBadData );
+    }
+}

src/modules/packagechooser/LoaderQueue.h

@@ -0,0 +1,77 @@
+/*
+ *   SPDX-FileCopyrightText: 2016 Luca Giambonini <almack@chakraos.org>
+ *   SPDX-FileCopyrightText: 2016 Lisa Vitolo     <shainer@chakraos.org>
+ *   SPDX-FileCopyrightText: 2017 Kyle Robbertze  <krobbertze@gmail.com>
+ *   SPDX-FileCopyrightText: 2017-2018 2020, Adriaan de Groot <groot@kde.org>
+ *   SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ *   Calamares is Free Software: see the License-Identifier above.
+ *
+ */
+
+#ifndef NETINSTALL_LOADERQUEUE_H
+#define NETINSTALL_LOADERQUEUE_H
+
+#include <QQueue>
+#include <QUrl>
+#include <QVariantList>
+
+class Config;
+class QNetworkReply;
+
+/** @brief Data about an entry in *groupsUrl*
+ *
+ * This can be a specific URL, or "local" which uses data stored
+ * in the configuration file itself.
+ */
+struct SourceItem
+{
+    QUrl url;
+    QVariantList data;
+
+    bool isUrl() const { return url.isValid(); }
+    bool isLocal() const { return !data.isEmpty(); }
+    bool isValid() const { return isUrl() || isLocal(); }
+    /** @brief Create a SourceItem
+     *
+     * If the @p groupsUrl is @c "local" then the *groups* key in
+     * the @p configurationMap is used as the source; otherwise the
+     * string is used as an actual URL.
+     */
+    static SourceItem makeSourceItem( const QString& groupsUrl, const QVariantMap& configurationMap );
+};
+
+/** @brief Queue of source items to load
+ *
+ * Queue things up by calling append() and then kick things off
+ * by calling load(). This will try to load the items, in order;
+ * the first one that succeeds will end the loading process.
+ *
+ * Signal done() is emitted when done (also when all of the items fail).
+ */
+class LoaderQueue : public QObject
+{
+    Q_OBJECT
+public:
+    LoaderQueue( Config* parent );
+
+    void append( SourceItem&& i );
+    int count() const { return m_queue.count(); }
+
+public Q_SLOTS:
+    void load();
+
+    void fetchNext();
+    void fetch( const QUrl& url );
+    void dataArrived();
+
+Q_SIGNALS:
+    void done();
+
+private:
+    QQueue< SourceItem > m_queue;
+    Config* m_config = nullptr;
+    QNetworkReply* m_reply = nullptr;
+};
+
+#endif