Ability to manually download the geolocation db

This removes the bundled geolocation db, saving about 8 MB of space.
It is now possible to manually download the database from the app settings.

Closes #172

.gitmodules

@@ -7,9 +7,6 @@
 [submodule "submodules/libpcap"]
 	path = submodules/libpcap
 	url = https://github.com/the-tcpdump-group/libpcap
-[submodule "app/src/main/res/raw"]
-	path = app/src/main/res/raw
-	url = https://github.com/emanuele-f/PCAPdroid_res
 [submodule "submodules/MaxMind-DB-Reader-java"]
 	path = submodules/MaxMind-DB-Reader-java
 	url = https://github.com/emanuele-f/MaxMind-DB-Reader-java

app/src/main/java/com/emanuelef/remote_capture/Geolocation.java

@@ -19,18 +19,21 @@
 
 package com.emanuelef.remote_capture;
 
+import android.annotation.SuppressLint;
 import android.content.Context;
 import android.util.Log;
 
+import androidx.annotation.Nullable;
+
 import com.emanuelef.remote_capture.model.Geomodel;
 import com.maxmind.db.Reader;
 
-import java.io.BufferedOutputStream;
 import java.io.File;
-import java.io.FileOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.InetAddress;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 
 /* A class to query geolocation info from IP addresses. */
 public class Geolocation {
@@ -46,67 +49,123 @@ public class Geolocation {
 
     @Override
     public void finalize() {
-        try {
-            mCountryReader.close();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        try {
-            mAsnReader.close();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
+        Utils.safeClose(mCountryReader);
+        Utils.safeClose(mAsnReader);
+        mCountryReader = null;
+        mAsnReader = null;
     }
 
     private void openDb() {
         try {
-            File countryFile = new File(mContext.getCacheDir() + "/dbip_country_lite.mmdb");
-            res_to_file(R.raw.dbip_country_lite, countryFile);
-            mCountryReader = new Reader(countryFile);
+            mCountryReader = new Reader(getCountryFile(mContext));
             Log.d(TAG, "Country DB loaded: " + mCountryReader.getMetadata());
 
-            File asnFile = new File(mContext.getCacheDir() + "/dbip_asn_lite.mmdb");
-            res_to_file(R.raw.dbip_asn_lite, asnFile);
-            mAsnReader = new Reader(asnFile);
+            mAsnReader = new Reader(getAsnFile(mContext));
             Log.d(TAG, "ASN DB loaded: " + mAsnReader.getMetadata());
         } catch (IOException e) {
-            e.printStackTrace();
-            throw new IllegalStateException();
+            Log.i(TAG, "Geolocation is not available");
         }
     }
 
-    // We need to get a File from the resource so that the Reader can mmap it
-    private void res_to_file(int resid, File dst) throws IOException {
-        try(InputStream is = mContext.getResources().openRawResource(resid)) {
-            try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(dst))) {
-                byte[] bytesIn = new byte[4096];
-                int read;
+    private static File getCountryFile(Context ctx) {
+        return new File(ctx.getFilesDir() + "/dbip_country_lite.mmdb");
+    }
 
-                while((read = is.read(bytesIn)) != -1)
-                    bos.write(bytesIn, 0, read);
+    private static File getAsnFile(Context ctx) {
+        return new File(ctx.getFilesDir() + "/dbip_asn_lite.mmdb");
+    }
+
+    public static Date getDbDate(File file) throws IOException {
+        try(Reader reader = new Reader(file)) {
+            return reader.getMetadata().getBuildDate();
+        }
+    }
+
+    public static @Nullable Date getDbDate(Context ctx) {
+        try {
+            return getDbDate(getCountryFile(ctx));
+        } catch (IOException ignored) {
+            return null;
+        }
+    }
+
+    public static long getDbSize(Context ctx) {
+        return getCountryFile(ctx).length() + getAsnFile(ctx).length();
+    }
+
+    @SuppressWarnings("ResultOfMethodCallIgnored")
+    public static void deleteDb(Context ctx) {
+        getCountryFile(ctx).delete();
+        getAsnFile(ctx).delete();
+    }
+
+    @SuppressLint("SimpleDateFormat")
+    public static boolean downloadDb(Context ctx) {
+        String dateid = new SimpleDateFormat("yyyy-MM").format(new Date());
+        String country_url = "https://download.db-ip.com/free/dbip-country-lite-" + dateid + ".mmdb.gz";
+        String asn_url = "https://download.db-ip.com/free/dbip-asn-lite-" + dateid + ".mmdb.gz";
+
+        try {
+            return downloadAndUnzip(ctx, "country", country_url, getCountryFile(ctx)) &&
+                    downloadAndUnzip(ctx, "asn", asn_url, getAsnFile(ctx));
+        } catch (IOException e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    private static boolean downloadAndUnzip(Context ctx, String label, String url, File dst) throws IOException {
+        File tmp_file = new File(ctx.getCacheDir() + "/geoip_db.zip");
+
+        boolean rv = Utils.downloadFile(url, tmp_file.getAbsolutePath());
+        if(!rv) {
+            Log.w(TAG, "Could not download " + label + " db from " +  url);
+            return false;
+        }
+
+        try(FileInputStream is = new FileInputStream(tmp_file.getAbsolutePath())) {
+            if(!Utils.ungzip(is, dst.getAbsolutePath())) {
+                Log.w(TAG, "ungzip of " + tmp_file + " failed");
+                return false;
             }
+
+            // Verify - throws IOException on error
+            getDbDate(dst);
+
+            return true;
+        } finally {
+            //noinspection ResultOfMethodCallIgnored
+            tmp_file.delete();
         }
     }
 
     public String getCountryCode(InetAddress addr) {
-        try {
-            Geomodel.CountryResult res = mCountryReader.get(addr, Geomodel.CountryResult.class);
-            if((res != null) && (res.country != null))
-                return res.country.isoCode;
-        } catch (IOException e) {
-            e.printStackTrace();
+        if(mCountryReader != null) {
+            try {
+                Geomodel.CountryResult res = mCountryReader.get(addr, Geomodel.CountryResult.class);
+                if ((res != null) && (res.country != null))
+                    return res.country.isoCode;
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
         }
+
+        // fallback
         return "";
     }
 
     public Geomodel.ASN getASN(InetAddress addr) {
-        try {
-            Geomodel.ASN res = mAsnReader.get(addr, Geomodel.ASN.class);
-            if(res != null)
-                return res;
-        } catch (IOException e) {
-            e.printStackTrace();
+        if(mAsnReader != null) {
+            try {
+                Geomodel.ASN res = mAsnReader.get(addr, Geomodel.ASN.class);
+                if (res != null)
+                    return res;
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
         }
+
+        // fallback
         return new Geomodel.ASN();
     }
 }

app/src/main/java/com/emanuelef/remote_capture/Utils.java

@@ -113,6 +113,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Random;
+import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -854,6 +855,21 @@ public class Utils {
         }
     }
 
+    public static boolean ungzip(InputStream is, String dst) {
+        try(GZIPInputStream gis = new GZIPInputStream(is)) {
+            try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(dst))) {
+                byte[] bytesIn = new byte[4096];
+                int read;
+                while ((read = gis.read(bytesIn)) != -1)
+                    bos.write(bytesIn, 0, read);
+            }
+            return true;
+        } catch (IOException e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
     public static boolean downloadFile(String _url, String path) {
         boolean has_contents = false;
 

app/src/main/java/com/emanuelef/remote_capture/activities/SettingsActivity.java

@@ -32,6 +32,9 @@ import android.util.Patterns;
 import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
 import androidx.annotation.NonNull;
+import androidx.fragment.app.Fragment;
+import androidx.fragment.app.FragmentManager;
+import androidx.fragment.app.FragmentTransaction;
 import androidx.preference.DropDownPreference;
 import androidx.preference.EditTextPreference;
 import androidx.preference.Preference;
@@ -43,6 +46,7 @@ import com.emanuelef.remote_capture.Billing;
 import com.emanuelef.remote_capture.PCAPdroid;
 import com.emanuelef.remote_capture.Utils;
 import com.emanuelef.remote_capture.MitmAddon;
+import com.emanuelef.remote_capture.fragments.GeoipSettings;
 import com.emanuelef.remote_capture.model.Prefs;
 import com.emanuelef.remote_capture.R;
 
@@ -52,7 +56,7 @@ import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.regex.Matcher;
 
-public class SettingsActivity extends BaseActivity {
+public class SettingsActivity extends BaseActivity implements PreferenceFragmentCompat.OnPreferenceStartFragmentCallback, FragmentManager.OnBackStackChangedListener {
     private static final String TAG = "SettingsActivity";
     private static final String ACTION_LANG_RESTART = "lang_restart";
     public static final String TARGET_PREF_EXTRA = "target_pref";
@@ -66,19 +70,53 @@ public class SettingsActivity extends BaseActivity {
 
         getSupportFragmentManager()
                 .beginTransaction()
-                .replace(R.id.settings_container, new SettingsFragment())
+                .replace(R.id.settings_container, new SettingsFragment(), "root")
                 .commit();
+
+        getSupportFragmentManager().addOnBackStackChangedListener(this);
+    }
+
+    @Override
+    public boolean onPreferenceStartFragment(@NonNull PreferenceFragmentCompat caller, @NonNull Preference pref) {
+        PreferenceFragmentCompat targetFragment = null;
+        Log.d(TAG, "startFragment: " + pref.getKey());
+
+        if(pref.getKey().equals("geolocation")) {
+            targetFragment = new GeoipSettings();
+            setTitle(R.string.geolocation);
+        }
+
+        if(targetFragment != null) {
+            getSupportFragmentManager()
+                    .beginTransaction()
+                    .replace(R.id.settings_container, targetFragment, pref.getKey())
+                    .setTransition(FragmentTransaction.TRANSIT_FRAGMENT_OPEN)
+                    .addToBackStack(pref.getKey())
+                    .commit();
+            return true;
+        }
+
+        return false;
+    }
+
+    @Override
+    public void onBackStackChanged() {
+        Fragment f = getSupportFragmentManager().findFragmentById(R.id.settings_container);
+        if(f instanceof SettingsFragment)
+            setTitle(R.string.title_activity_settings);
     }
 
     @Override
     public void onBackPressed() {
-        super.onBackPressed();
-
-        // Use a custom intent to provide "up" navigation after ACTION_LANG_RESTART took place
-        Intent intent = new Intent(this, MainActivity.class);
-        intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
-        startActivity(intent);
-        finish();
+        Fragment f = getSupportFragmentManager().findFragmentById(R.id.settings_container);
+        if(f instanceof SettingsFragment) {
+            // Use a custom intent to provide "up" navigation after ACTION_LANG_RESTART took place
+            Intent intent = new Intent(this, MainActivity.class);
+            intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
+            startActivity(intent);
+            finish();
+        } else
+            super.onBackPressed();
     }
 
     public static class SettingsFragment extends PreferenceFragmentCompat {

app/src/main/java/com/emanuelef/remote_capture/fragments/GeoipSettings.java

@@ -0,0 +1,116 @@
+/*
+ * This file is part of PCAPdroid.
+ *
+ * PCAPdroid is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * PCAPdroid is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with PCAPdroid.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright 2020-22 - Emanuele Faranda
+ */
+
+package com.emanuelef.remote_capture.fragments;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.Looper;
+import android.util.Log;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import androidx.appcompat.app.AlertDialog;
+import androidx.preference.Preference;
+import androidx.preference.PreferenceFragmentCompat;
+
+import com.emanuelef.remote_capture.Geolocation;
+import com.emanuelef.remote_capture.R;
+import com.emanuelef.remote_capture.Utils;
+
+import java.util.Date;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class GeoipSettings extends PreferenceFragmentCompat {
+    private static final String TAG = "GeoipSettings";
+    private Preference mStatus;
+    private Preference mDelete;
+
+    @Override
+    public void onCreatePreferences(@Nullable Bundle savedInstanceState, @Nullable String rootKey) {
+        setPreferencesFromResource(R.xml.geoip_preferences, rootKey);
+
+        mStatus = requirePreference("status");
+        mDelete = requirePreference("delete");
+        refreshStatus();
+
+        mDelete.setOnPreferenceClickListener(preference -> {
+            Geolocation.deleteDb(requireContext());
+            refreshStatus();
+            return true;
+        });
+
+        requirePreference("download")
+                .setOnPreferenceClickListener(preference -> {
+            downloadDatabases();
+            return true;
+        });
+    }
+
+    private void refreshStatus() {
+        Date builtDate = Geolocation.getDbDate(requireContext());
+        if(builtDate != null) {
+            String dateStr = Utils.formatEpochFull(requireContext(), builtDate.getTime() / 1000);
+            mStatus.setSummary("DB-IP Lite free\n" +
+                    String.format(getString(R.string.built_on), dateStr) + "\n" +
+                    String.format(getString(R.string.size_x), Utils.formatBytes(Geolocation.getDbSize(requireContext()))));
+        } else
+            mStatus.setSummary(R.string.geo_db_not_found);
+
+        mDelete.setVisible((builtDate != null));
+    }
+
+    private void downloadDatabases() {
+        ExecutorService executor = Executors.newSingleThreadExecutor();
+        Handler handler = new Handler(Looper.getMainLooper());
+
+        AlertDialog.Builder builder = new AlertDialog.Builder(requireContext());
+        builder.setTitle(R.string.downloading);
+        builder.setMessage(R.string.download_in_progress);
+
+        final AlertDialog alert = builder.create();
+        alert.setCanceledOnTouchOutside(false);
+        alert.show();
+
+        alert.setOnCancelListener(dialogInterface -> {
+            Log.i(TAG, "Abort download");
+            executor.shutdownNow();
+        });
+
+        executor.execute(() -> {
+            boolean result = Geolocation.downloadDb(requireContext());
+
+            handler.post(() -> {
+                if(!result)
+                    Utils.showToastLong(requireContext(), R.string.download_failed);
+
+                alert.dismiss();
+                refreshStatus();
+            });
+        });
+    }
+
+    private @NonNull
+    <T extends Preference> T requirePreference(String key) {
+        T pref = findPreference(key);
+        if(pref == null)
+            throw new IllegalStateException();
+        return pref;
+    }
+}

app/src/main/res/values/strings.xml

@@ -318,4 +318,16 @@
     <string name="display_as">Display as…</string>
     <string name="printable_text">Printable text</string>
     <string name="hexdump">Hexdump</string>
+    <string name="geolocation">Geolocation</string>
+    <string name="geolocation_summary">Add country and ASN info by performing offline lookups</string>
+    <string name="downloading">Downloading…</string>
+    <string name="download_in_progress">Download in progress, please wait</string>
+    <string name="download_failed">Download failed</string>
+    <string name="geo_db_not_found">Database not found. Geolocation is disabled</string>
+    <string name="database">Database</string>
+    <string name="built_on">Built on: %1$s</string>
+    <string name="geo_db_download">Tap to download the latest database. New databases are available monthly</string>
+    <string name="size_x">Size: %1$s</string>
+    <string name="geo_db_delete">Tap to delete the database and save space</string>
+    <string name="download">Download</string>
 </resources>

app/src/main/res/xml/geoip_preferences.xml

@@ -0,0 +1,26 @@
+<PreferenceScreen
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <Preference
+        app:key="status"
+        app:title="@string/database"
+        app:iconSpaceReserved="false"
+        android:summary="@string/geo_db_not_found">
+
+        <intent android:action="android.intent.action.VIEW"
+            android:data="https://db-ip.com/db/lite.php" />
+    </Preference>
+
+    <Preference
+        app:key="download"
+        app:title="@string/download"
+        android:summary="@string/geo_db_download"
+        app:iconSpaceReserved="false" />
+
+    <Preference
+        app:key="delete"
+        app:title="@string/delete"
+        android:summary="@string/geo_db_delete"
+        app:iconSpaceReserved="false" />
+</PreferenceScreen>

app/src/main/res/xml/root_preferences.xml

@@ -163,6 +163,13 @@
             app:defaultValue="system"
             app:useSimpleSummaryProvider="true"/>
 
+        <Preference
+            android:key="geolocation"
+            app:title="@string/geolocation"
+            app:summary="@string/geolocation_summary"
+            app:iconSpaceReserved="false"
+            app:fragment="com.emanuelef.remote_capture.fragments.GeoipSettings" />
+
         <SwitchPreference
             app:key="ipv6_enabled"
             app:title="@string/ipv6"