[locale] Fix QString-vs-char* confusion

- One sample JSON result
 - Two sample XML results

CMakeLists.txt

@@ -176,7 +176,7 @@ set( CALAMARES_TRANSLATION_LANGUAGES ar ast bg ca cs_CZ da de el en en_GB es_MX
 ### Bump version here
 set( CALAMARES_VERSION_MAJOR 3 )
 set( CALAMARES_VERSION_MINOR 1 )
-set( CALAMARES_VERSION_PATCH 12 )
+set( CALAMARES_VERSION_PATCH 13 )
 set( CALAMARES_VERSION_RC 0 )
 
 set( CALAMARES_VERSION ${CALAMARES_VERSION_MAJOR}.${CALAMARES_VERSION_MINOR}.${CALAMARES_VERSION_PATCH} )

Dockerfile

@@ -1,2 +1,2 @@
-FROM kdeneon/all
+FROM kdeneon/all:user
 RUN sudo apt-get update && sudo apt-get -y install build-essential cmake extra-cmake-modules gettext kio-dev libatasmart-dev libboost-python-dev libkf5config-dev libkf5coreaddons-dev libkf5i18n-dev libkf5iconthemes-dev libkf5parts-dev libkf5service-dev libkf5solid-dev libkpmcore-dev libparted-dev libpolkit-qt5-1-dev libqt5svg5-dev libqt5webkit5-dev libyaml-cpp-dev os-prober pkg-config python3-dev qtbase5-dev qtdeclarative5-dev qttools5-dev qttools5-dev-tools 

src/modules/locale/CMakeLists.txt

@@ -1,9 +1,26 @@
+find_package(ECM ${ECM_VERSION} NO_MODULE)
+if( ECM_FOUND AND BUILD_TESTING )
+    include( ECMAddTests )
+    find_package( Qt5 COMPONENTS Core Test REQUIRED )
+endif()
+
 include_directories( ${PROJECT_BINARY_DIR}/src/libcalamaresui )
 
+set( geoip_src GeoIP.cpp GeoIPFreeGeoIP.cpp )
+set( geoip_libs )
+
+find_package(Qt5 COMPONENTS Xml)
+if( Qt5Xml_FOUND )
+    list( APPEND geoip_src GeoIPXML.cpp )
+    list( APPEND geoip_libs Qt5::Xml )
+    add_definitions( -DHAVE_XML )
+endif()
+
 calamares_add_plugin( locale
     TYPE viewmodule
     EXPORT_MACRO PLUGINDLLEXPORT_PRO
     SOURCES
+        ${geoip_src}
         LCLocaleDialog.cpp
         LocaleConfiguration.cpp
         LocalePage.cpp
@@ -17,6 +34,28 @@ calamares_add_plugin( locale
     LINK_PRIVATE_LIBRARIES
         calamaresui
         Qt5::Network
+        ${geoip_libs}
         ${YAMLCPP_LIBRARY}
     SHARED_LIB
 )
+
+if( ECM_FOUND AND BUILD_TESTING )
+    ecm_add_test(
+            GeoIPTests.cpp
+            ${geoip_src}
+        TEST_NAME
+            geoiptest
+        LINK_LIBRARIES
+            calamaresui
+            Qt5::Network
+            Qt5::Test
+            ${geoip_libs}
+            ${YAMLCPP_LIBRARY}
+    )
+    set_target_properties( geoiptest PROPERTIES AUTOMOC TRUE )
+endif()
+
+if( BUILD_TESTING )
+    add_executable( test_geoip test_geoip.cpp ${geoip_src} )
+    target_link_libraries( test_geoip calamaresui Qt5::Network ${geoip_libs} ${YAMLCPP_LIBRARY} )
+endif()

src/modules/locale/GeoIP.cpp

@@ -0,0 +1,40 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "GeoIP.h"
+
+#include "utils/Logger.h"
+
+GeoIP::~GeoIP()
+{
+}
+
+GeoIP::RegionZonePair
+GeoIP::splitTZString( const QString& timezoneString )
+{
+    QStringList tzParts = timezoneString.split( '/', QString::SkipEmptyParts );
+    if ( tzParts.size() >= 2 )
+    {
+        cDebug() << "GeoIP reporting" << timezoneString;
+        QString region = tzParts.takeFirst();
+        QString zone = tzParts.join( '/' );
+        return qMakePair( region, zone );
+    }
+
+    return qMakePair( QString(), QString() );
+}

src/modules/locale/GeoIP.h

@@ -0,0 +1,56 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GEOIP_H
+#define GEOIP_H
+
+#include <QPair>
+#include <QString>
+#include <QUrl>
+
+class QByteArray;
+
+/**
+ * @brief Interface for GeoIP retrievers.
+ *
+ * A GeoIP retriever takes a configured URL (from the config file)
+ * and can handle the data returned from its interpretation of that
+ * configured URL, returning a region and zone.
+ */
+struct GeoIP
+{
+    using RegionZonePair = QPair<QString, QString>;
+
+    virtual ~GeoIP();
+
+    /** @brief Handle a (successful) request by interpreting the data.
+     *
+     * Should return a ( <zone>, <region> ) pair, e.g.
+     * ( "Europe", "Amsterdam" ). This is called **only** if the
+     * request to the fullUrl was successful; the handler
+     * is free to read as much, or as little, data as it
+     * likes. On error, returns a RegionZonePair with empty
+     * strings (e.g. ( "", "" ) ).
+     */
+    virtual RegionZonePair processReply( const QByteArray& ) = 0;
+
+    /** @brief Splits a region/zone string into a pair. */
+    static RegionZonePair splitTZString( const QString& s );
+} ;
+
+#endif

src/modules/locale/GeoIPFreeGeoIP.cpp

@@ -0,0 +1,55 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2014-2016, Teo Mrnjavac <teo@kde.org>
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "GeoIPFreeGeoIP.h"
+
+#include "utils/Logger.h"
+#include "utils/YamlUtils.h"
+
+#include <QByteArray>
+
+#include <yaml-cpp/yaml.h>
+
+GeoIP::RegionZonePair
+FreeGeoIP::processReply( const QByteArray& data )
+{
+    try
+    {
+        YAML::Node doc = YAML::Load( data );
+
+        QVariant var = CalamaresUtils::yamlToVariant( doc );
+        if ( !var.isNull() &&
+            var.isValid() &&
+            var.type() == QVariant::Map )
+        {
+            QVariantMap map = var.toMap();
+            if ( map.contains( "time_zone" ) &&
+                !map.value( "time_zone" ).toString().isEmpty() )
+            {
+                return splitTZString( map.value( "time_zone" ).toString() );
+            }
+        }
+    }
+    catch ( YAML::Exception& e )
+    {
+        CalamaresUtils::explainYamlException( e, data, "GeoIP data");
+    }
+
+    return qMakePair( QString(), QString() );
+}

src/modules/locale/GeoIPFreeGeoIP.h

@@ -0,0 +1,37 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GEOIPFREEGEOIP_H
+#define GEOIPFREEGEOIP_H
+
+#include "GeoIP.h"
+
+/** @brief GeoIP lookup via freegeoip.com
+ *
+ * This is the original implementation of GeoIP lookup,
+ * using the FreeGeoIP service, or similar which returns
+ * data in the same format.
+ *
+ * The data is assumed to be in JSON format with a time_zone attribute.
+ */
+struct FreeGeoIP : public GeoIP
+{
+    virtual RegionZonePair processReply( const QByteArray& );
+} ;
+
+#endif

src/modules/locale/GeoIPTests.cpp

@@ -0,0 +1,140 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "GeoIPTests.h"
+
+#include "GeoIPFreeGeoIP.h"
+#ifdef HAVE_XML
+#include "GeoIPXML.h"
+#endif
+
+#include <QtTest/QtTest>
+
+QTEST_GUILESS_MAIN( GeoIPTests )
+
+GeoIPTests::GeoIPTests()
+{
+}
+
+GeoIPTests::~GeoIPTests()
+{
+}
+
+void
+GeoIPTests::initTestCase()
+{
+}
+
+void
+GeoIPTests::testJSON()
+{
+    static const char data[] =
+        "{\"time_zone\":\"Europe/Amsterdam\"}";
+
+    FreeGeoIP handler;
+    auto tz = handler.processReply( data );
+
+    QCOMPARE( tz.first, QLatin1String( "Europe" ) );
+    QCOMPARE( tz.second, QLatin1String( "Amsterdam" ) );
+
+    // JSON is quite tolerant
+    tz = handler.processReply( "time_zone: \"Europe/Brussels\"" );
+    QCOMPARE( tz.second, QLatin1String( "Brussels" ) );
+
+    tz = handler.processReply( "time_zone: America/New_York\n" );
+    QCOMPARE( tz.first, QLatin1String( "America" ) );
+}
+
+void
+GeoIPTests::testJSONbad()
+{
+    static const char data[] = "time_zone: 1";
+
+    FreeGeoIP handler;
+    auto tz = handler.processReply( data );
+
+    tz = handler.processReply( data );
+    QCOMPARE( tz.first, QString() );
+
+    tz = handler.processReply( "" );
+    QCOMPARE( tz.first, QString() );
+
+    tz = handler.processReply( "<html><body>404 Forbidden</body></html>" );
+    QCOMPARE( tz.first, QString() );
+}
+
+
+void
+GeoIPTests::testXML()
+{
+    static const char data[] =
+        R"(<Response>
+  <Ip>85.150.1.1</Ip>
+  <Status>OK</Status>
+  <CountryCode>NL</CountryCode>
+  <CountryCode3>NLD</CountryCode3>
+  <CountryName>Netherlands</CountryName>
+  <RegionCode>None</RegionCode>
+  <RegionName>None</RegionName>
+  <City>None</City>
+  <ZipPostalCode/>
+  <Latitude>50.0</Latitude>
+  <Longitude>4.0</Longitude>
+  <AreaCode>0</AreaCode>
+  <TimeZone>Europe/Amsterdam</TimeZone>
+</Response>)";
+
+#ifdef HAVE_XML
+    XMLGeoIP handler;
+    auto tz = handler.processReply( data );
+
+    QCOMPARE( tz.first, QLatin1String( "Europe" ) );
+    QCOMPARE( tz.second, QLatin1String( "Amsterdam" ) );
+#endif
+}
+
+void
+GeoIPTests::testXML2()
+{
+    static const char data[] =
+        "<Response><TimeZone>America/North Dakota/Beulah</TimeZone></Response>";
+
+#ifdef HAVE_XML
+    XMLGeoIP handler;
+    auto tz = handler.processReply( data );
+
+    QCOMPARE( tz.first, QLatin1String( "America" ) );
+    QCOMPARE( tz.second, QLatin1String( "North Dakota/Beulah" ) );
+#endif
+}
+
+void
+GeoIPTests::testXMLbad()
+{
+#ifdef HAVE_XML
+    XMLGeoIP handler;
+    auto tz = handler.processReply( "{time_zone: \"Europe/Paris\"}" );
+    QCOMPARE( tz.first, QString() );
+
+    tz = handler.processReply( "<Response></Response>" );
+    QCOMPARE( tz.first, QString() );
+
+    tz = handler.processReply( "fnord<html/>" );
+    QCOMPARE( tz.first, QString() );
+#endif
+}

src/modules/locale/GeoIPTests.h

@@ -0,0 +1,40 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GEOIPTESTS_H
+#define GEOIPTESTS_H
+
+#include <QObject>
+
+class GeoIPTests : public QObject
+{
+    Q_OBJECT
+public:
+    GeoIPTests();
+    ~GeoIPTests() override;
+
+private Q_SLOTS:
+    void initTestCase();
+    void testJSON();
+    void testJSONbad();
+    void testXML();
+    void testXML2();
+    void testXMLbad();
+};
+
+#endif

src/modules/locale/GeoIPXML.cpp

@@ -0,0 +1,50 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "GeoIPXML.h"
+
+#include "utils/Logger.h"
+
+#include <QNetworkReply>
+#include <QtXml/QDomDocument>
+
+GeoIP::RegionZonePair
+XMLGeoIP::processReply( const QByteArray& data )
+{
+    QString domError;
+    int errorLine, errorColumn;
+
+    QDomDocument doc;
+    if ( doc.setContent( data, false, &domError, &errorLine, &errorColumn ) )
+    {
+        const auto tzElements = doc.elementsByTagName( "TimeZone" );
+        cDebug() << "GeoIP found" << tzElements.length() << "elements";
+        for ( int it = 0; it < tzElements.length(); ++it )
+        {
+            auto e = tzElements.at(it).toElement();
+            return splitTZString( e.text() );
+        }
+    }
+    else
+    {
+        cDebug() << "GeoIP XML data error:" << domError << "(line" << errorLine << errorColumn << ')';
+    }
+
+    return qMakePair( QString(), QString() );
+
+}

src/modules/locale/GeoIPXML.h

@@ -0,0 +1,36 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GEOIPXML_H
+#define GEOIPXML_H
+
+#include "GeoIP.h"
+
+/** @brief GeoIP lookup with XML data
+ *
+ * The data is assumed to be in XML format with a
+ *      <Response><TimeZone></TimeZone></Response>
+ * element, which contains the text (string) for the region/zone. This
+ * format is  expected by, e.g. the Ubiquity installer.
+ */
+struct XMLGeoIP : public GeoIP
+{
+    virtual RegionZonePair processReply( const QByteArray& );
+} ;
+
+#endif

src/modules/locale/LocaleConfiguration.cpp

@@ -43,9 +43,11 @@ LocaleConfiguration::fromLanguageAndLocation( const QString& languageLocale,
                                               const QStringList& availableLocales,
                                               const QString& countryCode )
 {
-    LocaleConfiguration lc = LocaleConfiguration();
+    LocaleConfiguration lc;
+
+    // Note that the documentation how this works is in packages.conf
     QString language = languageLocale.split( '_' ).first();
-    lc.myLanguageLocaleBcp47 = QLocale(language).bcp47Name();
+    lc.myLanguageLocaleBcp47 = QLocale(language).bcp47Name().toLower();
 
     QStringList linesForLanguage;
     for ( const QString &line : availableLocales )
@@ -288,7 +290,7 @@ LocaleConfiguration::isEmpty() const
 
 
 QMap< QString, QString >
-LocaleConfiguration::toMap()
+LocaleConfiguration::toMap() const
 {
     QMap< QString, QString > map;
 
@@ -324,3 +326,9 @@ LocaleConfiguration::toMap()
 
     return map;
 }
+
+QString
+LocaleConfiguration::toBcp47() const
+{
+    return myLanguageLocaleBcp47;
+}

src/modules/locale/LocaleConfiguration.h

@@ -35,16 +35,21 @@ public:
 
     bool isEmpty() const;
 
+    QMap< QString, QString > toMap() const;
+    // Note that the documentation how this works is in packages.conf
+    QString toBcp47() const;
+
     // These become all uppercase in locale.conf, but we keep them lowercase here to
     // avoid confusion with locale.h.
     QString lang, lc_numeric, lc_time, lc_monetary, lc_paper, lc_name, lc_address,
             lc_telephone, lc_measurement, lc_identification;
-    QString myLanguageLocaleBcp47;
-    QMap< QString, QString > toMap();
 
     // If the user has explicitly selected language (from the dialog)
     // or numbers format, set these to avoid implicit changes to them.
     bool explicit_lang, explicit_lc;
+
+private:
+    QString myLanguageLocaleBcp47;
 };
 
 #endif // LOCALECONFIGURATION_H

src/modules/locale/LocalePage.cpp

@@ -489,8 +489,10 @@ LocalePage::updateGlobalStorage()
             ->insert( "locationRegion", location.region );
     Calamares::JobQueue::instance()->globalStorage()
             ->insert( "locationZone", location.zone );
-    Calamares::JobQueue::instance()->globalStorage()
+
-            ->insert( "locale", m_selectedLocaleConfiguration.myLanguageLocaleBcp47);
+    const QString bcp47 = m_selectedLocaleConfiguration.toBcp47();
+    Calamares::JobQueue::instance()->globalStorage()->insert( "locale", bcp47 );
+    cDebug() << "Updated locale globals, BCP47=" << bcp47;
 
     // If we're in chroot mode (normal install mode), then we immediately set the
     // timezone on the live system.

src/modules/locale/LocaleViewStep.cpp

@@ -1,6 +1,7 @@
 /* === This file is part of Calamares - <http://github.com/calamares> ===
  *
  *   Copyright 2014-2016, Teo Mrnjavac <teo@kde.org>
+ *   Copyright 2018, Adriaan de Groot <groot.org>
  *
  *   Calamares is free software: you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -18,11 +19,17 @@
 
 #include "LocaleViewStep.h"
 
+#include "GeoIP.h"
+#include "GeoIPFreeGeoIP.h"
+#ifdef HAVE_XML
+#include "GeoIPXML.h"
+#endif
+#include "GlobalStorage.h"
+#include "JobQueue.h"
 #include "LocalePage.h"
+
 #include "timezonewidget/localeglobal.h"
 #include "widgets/WaitingWidget.h"
-#include "JobQueue.h"
-#include "GlobalStorage.h"
 
 #include "utils/CalamaresUtilsGui.h"
 #include "utils/Logger.h"
@@ -110,54 +117,52 @@ LocaleViewStep::setUpPage()
 void
 LocaleViewStep::fetchGeoIpTimezone()
 {
+    QString actualUrl( m_geoipUrl );
+    GeoIP *handler = nullptr;
+
+    if ( m_geoipStyle.isEmpty() || m_geoipStyle == "legacy" )
+    {
+        actualUrl.append( "/json/" );
+        handler = new FreeGeoIP;
+    }
+    else if ( m_geoipStyle == "json" )
+    {
+        handler = new FreeGeoIP;
+    }
+#if defined(HAVE_XML)
+    else if ( m_geoipStyle == "xml" )
+    {
+        handler = new XMLGeoIP;
+    }
+#endif
+    else
+    {
+        cDebug() << "WARNING: GeoIP Style" << m_geoipStyle << "is not recognized.";
+        setUpPage();
+        return;
+    }
+    cDebug() << "Fetching GeoIP data from" << actualUrl;
+
     QNetworkAccessManager *manager = new QNetworkAccessManager( this );
     connect( manager, &QNetworkAccessManager::finished,
             [=]( QNetworkReply* reply )
     {
         if ( reply->error() == QNetworkReply::NoError )
         {
-            QByteArray data = reply->readAll();
+            auto tz = handler->processReply( reply->readAll() );
-
+            if ( !tz.first.isEmpty() )
-            try
+                m_startingTimezone = tz;
-            {
+            else
-                YAML::Node doc = YAML::Load( data );
+                cDebug() << "WARNING: GeoIP lookup at" << reply->url() << "failed.";
-
-                QVariant var = CalamaresUtils::yamlToVariant( doc );
-                if ( !var.isNull() &&
-                    var.isValid() &&
-                    var.type() == QVariant::Map )
-                {
-                    QVariantMap map = var.toMap();
-                    if ( map.contains( "time_zone" ) &&
-                        !map.value( "time_zone" ).toString().isEmpty() )
-                    {
-                        QString timezoneString = map.value( "time_zone" ).toString();
-                        QStringList tzParts = timezoneString.split( '/', QString::SkipEmptyParts );
-                        if ( tzParts.size() >= 2 )
-                        {
-                            cDebug() << "GeoIP reporting" << timezoneString;
-                            QString region = tzParts.takeFirst();
-                            QString zone = tzParts.join( '/' );
-                            m_startingTimezone = qMakePair( region, zone );
-                        }
-                    }
-                }
-            }
-            catch ( YAML::Exception& e )
-            {
-                CalamaresUtils::explainYamlException( e, data, "GeoIP data");
-            }
         }
-
+        delete handler;
         reply->deleteLater();
         manager->deleteLater();
         setUpPage();
     } );
 
     QNetworkRequest request;
-    QString requestUrl = QString( "%1/json" )
+    request.setUrl( QUrl::fromUserInput( actualUrl ) );
-                         .arg( QUrl::fromUserInput( m_geoipUrl ).toString() );
-    request.setUrl( QUrl( requestUrl ) );
     request.setAttribute( QNetworkRequest::FollowRedirectsAttribute, true );
     manager->get( request );
 }
@@ -293,4 +298,10 @@ LocaleViewStep::setConfigurationMap( const QVariantMap& configurationMap )
     {
         m_geoipUrl = configurationMap.value( "geoipUrl" ).toString();
     }
+    if ( configurationMap.contains( "geoipStyle" ) &&
+         configurationMap.value( "geoipStyle" ).type() == QVariant::String &&
+         !configurationMap.value( "geoipStyle" ).toString().isEmpty() )
+    {
+        m_geoipStyle = configurationMap.value( "geoipStyle" ).toString();
+    }
 }

src/modules/locale/LocaleViewStep.h

@@ -76,6 +76,7 @@ private:
     QPair< QString, QString > m_startingTimezone;
     QString m_localeGenPath;
     QString m_geoipUrl;
+    QString m_geoipStyle;
 
     QList< Calamares::job_ptr > m_jobs;
 };

src/modules/locale/locale.conf

@@ -1,7 +1,56 @@
 ---
+# The starting timezone (e.g. the pin-on-the-map) when entering
+# the locale page can be set through keys *region* and *zone*.
+# If either is not set, defaults to America/New_York.
+#
 region:                     "America"
 zone:                       "New_York"
 
-# GeoIP settings. Leave commented out to disable GeoIP.
+# Some distros come with a meaningfully commented and easy to parse
+# `/etc/locale.gen`, and others ship a separate file
+# `/usr/share/i18n/SUPPORTED` with a clean list of supported locales.
+# We first try SUPPORTED, and if it doesn't exist, we fall back
+# to parsing the lines from `locale.gen`. For distro's that ship
+# the `locale.gen` file installed elsewhere, the key *localeGenPath*
+# can be used to specify where it is. If not set, the default
+# `/etc/locale.gen` is used.
+#
 #localeGenPath:             "/etc/locale.gen"
+
+# GeoIP settings. Leave commented out to disable GeoIP.
+#
+# An HTTP request is made to *geoipUrl* -- depending on the geoipStyle,
+# the URL may be modified before use. The request should return
+# valid data in a suitable format, depending on geoipStyle;
+# generally this includes a string value with the timezone
+# in <region>/<zone> format.
+#
+# Note that this example URL works, but the service is shutting
+# down in June 2018.
+#
+# Suitable JSON data looks like
+# ```
+#    {"time_zone":"America/New_York"}
+# ```
+# Suitable XML data looks like
+# ```
+#    <Response><TimeZone>Europe/Brussels</TimeZone></Response>
+# ```
+#
 #geoipUrl:                  "freegeoip.net"
+
+# GeoIP style. Leave commented out for the "legacy" interpretation.
+# This setting only makes sense if geoipUrl is set, enabliing geoIP.
+#
+# Possible values are:
+#   unset       same as "legacy"
+#   blank       same as "legacy"
+#   "legacy"    appends "/json" to geoipUrl, above, and uses JSON format
+#               (which is what freegeoip.net provides there).
+#   "json"      URL is not modified, uses JSON format.
+#   "xml"       URL is not modified, uses XML format.
+#
+# The JSON format is provided by freegeoip.net, but that service is
+# shutting down in June 2018. There are other providers with the same
+# format. XML format is provided for Ubiquity.
+#geoipStyle:    "legacy"

src/modules/locale/test_geoip.cpp

@@ -0,0 +1,73 @@
+/* === This file is part of Calamares - <http://github.com/calamares> ===
+ *
+ *   Copyright 2018, Adriaan de Groot <groot@kde.org>
+ *
+ *   Calamares is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Calamares is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * This is a test-application that does one GeoIP parse.
+ */
+
+#include <iostream>
+
+#include "GeoIPFreeGeoIP.h"
+#ifdef HAVE_XML
+#include "GeoIPXML.h"
+#endif
+
+using std::cerr;
+
+int main(int argc, char** argv)
+{
+    if (argc != 2)
+    {
+        cerr << "Usage: curl url | test_geoip <format>\n";
+        return 1;
+    }
+
+    GeoIP* handler = nullptr;
+    if ( QLatin1String( "json" ) == argv[1] )
+        handler = new FreeGeoIP;
+#ifdef HAVE_XML
+    else if ( QLatin1String( "xml" ) == argv[1] )
+        handler = new XMLGeoIP;
+#endif
+
+    if ( !handler )
+    {
+        cerr << "Unknown format '" << argv[1] << "'\n";
+        return 1;
+    }
+
+    QByteArray ba;
+    while( !std::cin.eof() ) {
+        char arr[1024];
+        std::cin.read(arr,sizeof(arr));
+        int s = std::cin.gcount();
+        ba.append(arr, s);
+    }
+
+    auto tz = handler->processReply( ba );
+    if ( tz.first.isEmpty() )
+    {
+        std::cout << "No TimeZone determined from input.\n";
+    }
+    else
+    {
+        std::cout << "TimeZone Region=" << tz.first.toLatin1().constData() << "\nTimeZone Zone=" << tz.second.toLatin1().constData() << '\n';
+    }
+
+    return 0;
+}

src/modules/packages/main.py

@@ -318,7 +318,10 @@ def subst_locale(plist):
     """
     locale = libcalamares.globalstorage.value("locale")
     if not locale:
-        return plist
+        # It is possible to skip the locale-setting entirely.
+        # Then pretend it is "en", so that {LOCALE}-decorated
+        # package names are removed from the list.
+        locale = "en"
 
     ret = []
     for packagedata in plist:
@@ -364,20 +367,20 @@ def run_operations(pkgman, entry):
     global group_packages, completed_packages, mode_packages
 
     for key in entry.keys():
-        entry[key] = subst_locale(entry[key])
+        package_list = subst_locale(entry[key])
-        group_packages = len(entry[key])
+        group_packages = len(package_list)
         if key == "install":
             _change_mode(INSTALL)
-            if all([isinstance(x, str) for x in entry[key]]):
+            if all([isinstance(x, str) for x in package_list]):
-                pkgman.install(entry[key])
+                pkgman.install(package_list)
             else:
-                for package in entry[key]:
+                for package in package_list:
                     pkgman.install_package(package)
         elif key == "try_install":
             _change_mode(INSTALL)
             # we make a separate package manager call for each package so a
             # single failing package won't stop all of them
-            for package in entry[key]:
+            for package in package_list:
                 try:
                     pkgman.install_package(package)
                 except subprocess.CalledProcessError:
@@ -386,10 +389,10 @@ def run_operations(pkgman, entry):
                     libcalamares.utils.debug(warn_text)
         elif key == "remove":
             _change_mode(REMOVE)
-            pkgman.remove(entry[key])
+            pkgman.remove(package_list)
         elif key == "try_remove":
             _change_mode(REMOVE)
-            for package in entry[key]:
+            for package in package_list:
                 try:
                     pkgman.remove([package])
                 except subprocess.CalledProcessError:
@@ -398,9 +401,9 @@ def run_operations(pkgman, entry):
                     libcalamares.utils.debug(warn_text)
         elif key == "localInstall":
             _change_mode(INSTALL)
-            pkgman.install(entry[key], from_local=True)
+            pkgman.install(package_list, from_local=True)
 
-        completed_packages += len(entry[key])
+        completed_packages += len(package_list)
         libcalamares.job.setprogress(completed_packages * 1.0 / total_packages)
         libcalamares.utils.debug(pretty_name())
 
@@ -444,7 +447,7 @@ def run():
     completed_packages = 0
     for op in operations:
         for packagelist in op.values():
-            total_packages += len(packagelist)
+            total_packages += len(subst_locale(packagelist))
 
     if not total_packages:
         # Avoids potential divide-by-zero in progress reporting

src/modules/packages/packages.conf

@@ -76,7 +76,7 @@ update_db: true
 #     pre-script: touch /tmp/installing-vi
 #     post-script: rm -f /tmp/installing-vi
 #
-# The pre- and post-scripts are optional, but not both optional: using
+# The pre- and post-scripts are optional, but you cannot leave both out: using
 # "package: vi" with neither script option will trick Calamares into
 # trying to install a package named "package: vi", which is unlikely to work.
 #
@@ -84,11 +84,16 @@ update_db: true
 # packages for software based on the selected system locale. By including
 # the string LOCALE in the package name, the following happens:
 #
-# - if the system locale is English (generally US English; en_GB is a valid
+# - if the system locale is English (any variety), then the package is not
-#   localization), then the package is not installed at all,
+#   installed at all,
-# - otherwise $LOCALE or ${LOCALE} is replaced by the Bcp47 name of the selected
+# - otherwise $LOCALE or ${LOCALE} is replaced by the **lower-cased** BCP47
-#   system locale, e.g. nl_BE. Note that just plain LOCALE will not be replaced,
+#   name of the **language** part of the selected system locale (not the
-#   so foo-LOCALE will be unchanged, while foo-$LOCALE will be changed.
+#   country/region/dialect part), e.g. selecting *nl_BE* will use *nl*
+#   here.
+#
+# Take care that just plain LOCALE will not be replaced, so foo-LOCALE will
+# be left unchanged, while foo-$LOCALE will be changed. However, foo-LOCALE
+# **will** be removed from the list of packages, if English is selected.
 #
 # The following installs localizations for vi, if they are relevant; if
 # there is no localization, installation continues normally.
@@ -127,6 +132,7 @@ update_db: true
 operations:
   - install:
     - vi
+    - vi-${LOCALE}
     - wget
     - binutils
   - remove:

src/modules/plasmalnf/CMakeLists.txt

@@ -4,8 +4,13 @@ find_package(ECM ${ECM_VERSION} REQUIRED NO_MODULE)
 # needs a runtime support component (which we don't test for).
 set( lnf_ver 5.41 )
 
+find_package( KF5Config ${lnf_ver} )
 find_package( KF5Plasma ${lnf_ver} )
 find_package( KF5Package ${lnf_ver} )
+set_package_properties(
+    KF5Config PROPERTIES
+    PURPOSE "For finding default Plasma Look-and-Feel"
+)
 set_package_properties(
     KF5Plasma PROPERTIES
     PURPOSE "For Plasma Look-and-Feel selection"
@@ -16,11 +21,19 @@ set_package_properties(
 )
 
 if ( KF5Plasma_FOUND AND KF5Package_FOUND )
-    find_package( KF5 ${lnf_ver} REQUIRED CoreAddons Plasma Package )
+    if ( KF5Config_FOUND )
+        set( option_kf5 Config )
+        set( option_defs WITH_KCONFIG )
+        # set( option_libs KF5::Config )  # Not needed anyway
+    endif()
+
+    find_package( KF5 ${lnf_ver} REQUIRED CoreAddons Plasma Package ${option_kf5} )
 
     calamares_add_plugin( plasmalnf
         TYPE viewmodule
         EXPORT_MACRO PLUGINDLLEXPORT_PRO
+        COMPILE_DEFINITIONS
+            ${option_defs}
         SOURCES
             PlasmaLnfViewStep.cpp
             PlasmaLnfPage.cpp
@@ -32,6 +45,7 @@ if ( KF5Plasma_FOUND AND KF5Package_FOUND )
             page_plasmalnf.ui
         LINK_PRIVATE_LIBRARIES
             calamaresui
+            ${option_libs}
             KF5::Package
             KF5::Plasma
         SHARED_LIB

src/modules/plasmalnf/PlasmaLnfPage.cpp

@@ -23,6 +23,8 @@
 #include "utils/Logger.h"
 #include "utils/Retranslator.h"
 
+#include <QAbstractButton>
+
 #include <KPackage/Package>
 #include <KPackage/PackageLoader>
 
@@ -55,13 +57,18 @@ static ThemeInfoList plasma_themes()
 PlasmaLnfPage::PlasmaLnfPage( QWidget* parent )
     : QWidget( parent )
     , ui( new Ui::PlasmaLnfPage )
+    , m_showAll( false )
     , m_buttonGroup( nullptr )
 {
     ui->setupUi( this );
     CALAMARES_RETRANSLATE(
     {
         ui->retranslateUi( this );
-        ui->generalExplanation->setText( tr( "Please choose a look-and-feel for the KDE Plasma Desktop. You can also skip this step and configure the look-and-feel once the system is installed." ) );
+        ui->generalExplanation->setText( tr(
+            "Please choose a look-and-feel for the KDE Plasma Desktop. "
+            "You can also skip this step and configure the look-and-feel "
+            "once the system is installed. Clicking on a look-and-feel "
+            "selection will give you a live preview of that look-and-feel.") );
         updateThemeNames();
         fillUi();
     }
@@ -75,10 +82,18 @@ PlasmaLnfPage::setLnfPath( const QString& path )
 }
 
 void
-PlasmaLnfPage::setEnabledThemes(const ThemeInfoList& themes)
+PlasmaLnfPage::setEnabledThemes(const ThemeInfoList& themes, bool showAll )
 {
     m_enabledThemes = themes;
 
+    if ( showAll )
+    {
+        auto plasmaThemes = plasma_themes();
+        for ( auto& installed_theme : plasmaThemes )
+            if ( !m_enabledThemes.findById( installed_theme.id ) )
+                m_enabledThemes.append( installed_theme );
+    }
+
     updateThemeNames();
     winnowThemes();
     fillUi();
@@ -87,9 +102,18 @@ PlasmaLnfPage::setEnabledThemes(const ThemeInfoList& themes)
 void
 PlasmaLnfPage::setEnabledThemesAll()
 {
-    setEnabledThemes( plasma_themes() );
+    // Don't need to set showAll=true, because we're already passing in
+    // the complete list of installed themes.
+    setEnabledThemes( plasma_themes(), false );
 }
 
+void
+PlasmaLnfPage::setPreselect( const QString& id )
+{
+    m_preselect = id;
+    if ( !m_enabledThemes.isEmpty() )
+        fillUi();
+}
 
 void PlasmaLnfPage::updateThemeNames()
 {
@@ -162,6 +186,11 @@ void PlasmaLnfPage::fillUi()
         {
             theme.widget->updateThemeName( theme );
         }
+        if ( theme.id == m_preselect )
+        {
+            const QSignalBlocker b( theme.widget->button() );
+            theme.widget->button()->setChecked( true );
+        }
         ++c;
     }
 }

src/modules/plasmalnf/PlasmaLnfPage.h

@@ -46,10 +46,17 @@ public:
     explicit PlasmaLnfPage( QWidget* parent = nullptr );
 
     void setLnfPath( const QString& path );
-    /** @brief enable only the listed themes. */
+    /** @brief enable only the listed themes.
-    void setEnabledThemes( const ThemeInfoList& themes );
+     *
+     * Shows the listed @p themes with full information (e.g. screenshot).
+     * If @p showAll is true, then also show all installed themes
+     * not explicitly listed (without a screenshot).
+     */
+    void setEnabledThemes( const ThemeInfoList& themes, bool showAll );
     /** @brief enable all installed plasma themes. */
     void setEnabledThemesAll();
+    /** @brief set which theme is to be preselected. */
+    void setPreselect( const QString& id );
 
 signals:
     void plasmaThemeSelected( const QString& id );
@@ -64,6 +71,8 @@ private:
 
     Ui::PlasmaLnfPage* ui;
     QString m_lnfPath;
+    QString m_preselect;
+    bool m_showAll;  // If true, don't winnow according to enabledThemes
     ThemeInfoList m_enabledThemes;
 
     QButtonGroup *m_buttonGroup;

src/modules/plasmalnf/PlasmaLnfViewStep.cpp

@@ -26,8 +26,25 @@
 #include <QProcess>
 #include <QVariantMap>
 
+#ifdef WITH_KCONFIG
+#include <KConfigGroup>
+#include <KSharedConfig>
+#endif
+
 CALAMARES_PLUGIN_FACTORY_DEFINITION( PlasmaLnfViewStepFactory, registerPlugin<PlasmaLnfViewStep>(); )
 
+static QString
+currentPlasmaTheme()
+{
+#ifdef WITH_KCONFIG
+    KConfigGroup cg( KSharedConfig::openConfig( QStringLiteral( "kdeglobals" ) ), "KDE" );
+    return cg.readEntry( "LookAndFeelPackage", QString() );
+#else
+    cWarning() << "No KConfig support, cannot determine Plasma theme.";
+    return QString();
+#endif
+}
+
 PlasmaLnfViewStep::PlasmaLnfViewStep( QObject* parent )
     : Calamares::ViewStep( parent )
     , m_widget( new PlasmaLnfPage )
@@ -137,10 +154,22 @@ PlasmaLnfViewStep::setConfigurationMap( const QVariantMap& configurationMap )
         liveUser = configurationMap.value( "liveuser" ).toString();
     m_liveUser = liveUser;
 
+    QString preselect;
+    if ( configurationMap.contains( "preselect" ) && configurationMap.value( "preselect" ).type() == QVariant::String )
+        preselect = configurationMap.value( "preselect" ).toString();
+    if ( preselect == QStringLiteral( "*" ) )
+        preselect = currentPlasmaTheme();
+    if ( !preselect.isEmpty() )
+        m_widget->setPreselect( preselect );
+
+    bool showAll( false );
+    if ( configurationMap.contains( "showAll" ) && configurationMap.value( "showAll" ).type() == QVariant::Bool )
+        showAll = configurationMap.value( "showAll" ).toBool();
+
     if ( configurationMap.contains( "themes" ) &&
         configurationMap.value( "themes" ).type() == QVariant::List )
     {
-        ThemeInfoList allThemes;
+        ThemeInfoList listedThemes;
         auto themeList = configurationMap.value( "themes" ).toList();
         // Create the ThemInfo objects for the listed themes; information
         // about the themes from Plasma (e.g. human-readable name and description)
@@ -149,14 +178,14 @@ PlasmaLnfViewStep::setConfigurationMap( const QVariantMap& configurationMap )
             if ( i.type() == QVariant::Map )
             {
                 auto iv = i.toMap();
-                allThemes.append( ThemeInfo( iv.value( "theme" ).toString(), iv.value( "image" ).toString() ) );
+                listedThemes.append( ThemeInfo( iv.value( "theme" ).toString(), iv.value( "image" ).toString() ) );
             }
             else if ( i.type() == QVariant::String )
-                allThemes.append( ThemeInfo( i.toString() ) );
+                listedThemes.append( ThemeInfo( i.toString() ) );
 
-        if ( allThemes.length() == 1 )
+        if ( listedThemes.length() == 1 )
             cDebug() << "WARNING: only one theme enabled in plasmalnf";
-        m_widget->setEnabledThemes( allThemes );
+        m_widget->setEnabledThemes( listedThemes, showAll );
     }
     else
         m_widget->setEnabledThemesAll();  // All of them

src/modules/plasmalnf/PlasmaLnfViewStep.h

@@ -61,9 +61,9 @@ public slots:
 
 private:
     PlasmaLnfPage* m_widget;
-    QString m_lnfPath;
+    QString m_lnfPath;  // Path to the lnf tool
-    QString m_themeId;
+    QString m_themeId;  // Id of selected theme
-    QString m_liveUser;
+    QString m_liveUser; // Name of the live user (for OEM mode)
 };
 
 CALAMARES_PLUGIN_FACTORY_DECLARATION( PlasmaLnfViewStepFactory )

src/modules/plasmalnf/ThemeWidget.cpp

@@ -62,7 +62,7 @@ ThemeWidget::ThemeWidget(const ThemeInfo& info, QWidget* parent)
     layout->addWidget( image_label, 1 );
     layout->addWidget( m_description, 3 );
 
-    connect( m_check, &QRadioButton::clicked, this, &ThemeWidget::clicked );
+    connect( m_check, &QRadioButton::toggled, this, &ThemeWidget::clicked );
 }
 
 void

src/modules/plasmalnf/plasmalnf.conf

@@ -13,8 +13,9 @@ lnftool:	"/usr/bin/lookandfeeltool"
 
 # You can limit the list of Plasma look-and-feel themes by listing ids
 # here. If this key is not present, all of the installed themes are listed.
-# If the key is present, only installed themes that are *also* included
+# If the key is present, only installed themes that are **also** included
-# in the list are shown (could be none!).
+# in the list are shown (could be none!). See the *showAll* key, below,
+# to change that.
 #
 # Themes may be listed by id, (e.g. fluffy-bunny, below) or as a theme
 # and an image (e.g. breeze) which will be used to show a screenshot.
@@ -27,3 +28,26 @@ themes:
     - theme: org.kde.breezedark.desktop
       image: "breeze-dark.png"
     - org.kde.fluffy-bunny.desktop
+
+# If *showAll* is true, then all installed themes are shown in the
+# UI for selection, even if they are not listed in *themes*. This
+# allows selection of all themes even while not all of them are
+# listed in *themes* -- which is useful to show screenshots for those
+# you do have a screenshot for.
+showAll: false
+
+# You can pre-select one of the themes; it is not applied
+# immediately, but its radio-button is switched on to indicate
+# that that is the theme (that is most likely) currently in use.
+# Do this only on Live images where you are reasonably sure
+# that the user is not going to change the theme out from under
+# themselves before running the installer.
+#
+# If this key is present, its value should be the id of the theme
+# which should be pre-selected. If absent, empty, or the pre-selected
+# theme is not found on the live system, no theme will be pre-selected.
+#
+# As a special setting, use "*", to try to find the currently-
+# selected theme by reading the Plasma configuration. This requires
+# KF5::Config at build- and run-time.
+preselect: "*"

src/modules/welcome/welcome.conf

@@ -25,6 +25,6 @@ requirements:
     # If any of these conditions are not met, the user cannot
     # continue past the welcome page.
     required:
-        - storage
+        # - storage
         - ram
-        - root
+        # - root