Add a large shapefile to map in python using folium

Question

I am displaying a folium map in my application using python, PyQt5 and Qt designer. Since there is no map widget in Qt designer, I add a general widget and then promote it to my custom map widget. It all works fine. Here is the python code for my promoted widget:

import io

import folium

from PyQt5 import QtWebEngineWidgets
from PyQt5.QtWidgets import *

class LeafWidget (QWidget):
    def __init__(self, parent=None):
        QWidget.__init__(self, parent)
        m = folium.Map(
            location=[40, -120] , zoom_start=10
        )
        self.view = QtWebEngineWidgets.QWebEngineView()
       
        data = io.BytesIO()
       
        m.save(data, close_file=False)
        self.view.setHtml(data.getvalue().decode())
        self.layout = QVBoxLayout(self)
        self.layout.addWidget(self.view)
        self.show()

This works fine and I can see the map in my application.

I am also trying to display a GIS shapefile on top of this map. I have done some research and it seems like I cannot add GIS shapefile (.shp) directly to a folium map. So, I try to convert it to json first and then add the json on top of the map. I modified my code as below to add the .shp file to map:

import io

import folium
import os.path

from PyQt5 import QtWebEngineWidgets
from PyQt5.QtWidgets  import *
import geopandas as gpd

class LeafWidget (QWidget):
    def __init__(self, parent=None):
        QWidget.__init__(self, parent)
        m = folium.Map(
            location=[40, -120] , zoom_start=10
        )
        self.view = QtWebEngineWidgets.QWebEngineView()
        # converting shp to geojson
        shp_file = gpd.read_file('input/2015_loaded_NoCC.shp')
        shp_file.to_file('myshpfile.json', driver='GeoJSON')
        shp = os.path.join('', 'myshpfile.json')
        data = io.BytesIO()
        folium.GeoJson(shp).add_to(m)
        m.save(data, close_file=False)
        self.view.setHtml(data.getvalue().decode())
        self.layout = QVBoxLayout(self)
        self.layout.addWidget(self.view)
        self.show()

but now my map doesn't show up at all. It's just an empty space with no errors in the console or error log. If I save the map as an HTML file using "m.save('map.html')" though, it does save the file and when I open it, it displays the json file on the map, but for some reason, the way I am doing it to show the map in my application is not working after adding the shp-->json file. What am I doing wrong?

Answer 1

As already pointed out in these questions( 1 and 2 ) and in the official docs :

void QWebEnginePage::setHtml(const QString &html, const QUrl &baseUrl = QUrl())

Sets the content of this page to html. baseUrl is optional and used to resolve relative URLs in the document, such as referenced images or stylesheets.

The html is loaded immediately; external objects are loaded asynchronously.

If a script in the html runs longer than the default script timeout (currently 10 seconds), for example due to being blocked by a modal JavaScript alert dialog, this method will return as soon as possible after the timeout and any subsequent html will be loaded asynchronously.

When using this method, the web engine assumes that external resources, such as JavaScript programs or style sheets, are encoded in UTF-8 unless otherwise specified. For example, the encoding of an external script can be specified through the charset attribute of the HTML script tag. It is also possible for the encoding to be specified by the web server.

This is a convenience function equivalent to setContent(html, "text/html", baseUrl).

Note: This method will not affect session or global history for the page.

Warning: This function works only for HTML, for other mime types (such as XHTML and SVG) setContent() should be used instead.

Warning: The content will be percent encoded before being sent to the renderer via IPC. This may increase its size. The maximum size of the percent encoded content is 2 megabytes minus 30 bytes.

_{(emphasis mine)}

setHtml() does not support content greater than 2MB, so in your particular case there are 2 solutions:

• Save the folium map in an html file:

   import io import os from PyQt5 import QtCore, QtWidgets, QtWebEngineWidgets CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) class LeafWidget(QtWidgets.QWidget): def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) self.view = QtWebEngineWidgets.QWebEngineView() shp_filename = os.path.join(CURRENT_DIR, "input", "2015_loaded_NoCC.shp") shp_file = gpd.read_file(shp_filename) shp_file_json_str = shp_file.to_json() m = folium.Map(location=[40, -120], zoom_start=10) folium.GeoJson(shp_file_json_str).add_to(m) tmp_file = QtCore.QTemporaryFile("XXXXXX.html", self) if tmp_file.open(): m.save(tmp_file.fileName()) url = QtCore.QUrl.fromLocalFile(tmp_file.fileName()) self.view.load(url) lay = QtWidgets.QVBoxLayout(self) lay.addWidget(self.view) def main(): app = QtWidgets.QApplication([]) w = LeafWidget() w.show() app.exec_() if __name__ == "__main__": main()

• Use a QWebEngineUrlSchemeHandler to return the html:

  qfolium.py

   import json import io from PyQt5 import QtCore, QtWebEngineCore, QtWebEngineWidgets class FoliumSchemeHandler(QtWebEngineCore.QWebEngineUrlSchemeHandler): def __init__(self, app): super().__init__(app) self.m_app = app def requestStarted(self, request): url = request.requestUrl() name = url.host() m = self.m_app.process(name, url.query()) if m is None: request.fail(QtWebEngineCore.QWebEngineUrlRequestJob.UrlNotFound) return data = io.BytesIO() m.save(data, close_file=False) raw_html = data.getvalue() buf = QtCore.QBuffer(parent=self) request.destroyed.connect(buf.deleteLater) buf.open(QtCore.QIODevice.WriteOnly) buf.write(raw_html) buf.seek(0) buf.close() request.reply(b"text/html", buf) class FoliumApplication(QtCore.QObject): scheme = b"folium" def __init__(self, parent=None): super().__init__(parent) scheme = QtWebEngineCore.QWebEngineUrlScheme(self.scheme) QtWebEngineCore.QWebEngineUrlScheme.registerScheme(scheme) self.m_functions = dict() def init_handler(self, profile=None): if profile is None: profile = QtWebEngineWidgets.QWebEngineProfile.defaultProfile() handler = profile.urlSchemeHandler(self.scheme) if handler is not None: profile.removeUrlSchemeHandler(handler) self.m_handler = FoliumSchemeHandler(self) profile.installUrlSchemeHandler(self.scheme, self.m_handler) def register(self, name): def decorator(f): self.m_functions[name] = f return f return decorator def process(self, name, query): f = self.m_functions.get(name) if f is None: print("not found") return items = QtCore.QUrlQuery(query).queryItems() params_json = dict(items).get("json", None) if params_json is not None: return f(**json.loads(params_json)) return f() def create_url(self, name, params=None): url = QtCore.QUrl() url.setScheme(self.scheme.decode()) url.setHost(name) if params is not None: params_json = json.dumps(params) query = QtCore.QUrlQuery() query.addQueryItem("json", params_json) url.setQuery(query) return url

  main.py

  import io import os import folium from PyQt5 import QtCore, QtWidgets, QtWebEngineWidgets import geopandas as gpd from qfolium import FoliumApplication CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) folium_app = FoliumApplication() @folium_app.register("load_shapefile") def load_shapefile(latitude, longitude, zoom_start, shp_filename): shp_file = gpd.read_file(shp_filename) shp_file_json_str = shp_file.to_json() m = folium.Map( location=[latitude, longitude], zoom_start=zoom_start ) folium.GeoJson(shp_file_json_str).add_to(m) print(m) return m class LeafWidget(QtWidgets.QWidget): def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) self.view = QtWebEngineWidgets.QWebEngineView() lay = QtWidgets.QVBoxLayout(self) lay.addWidget(self.view) self.resize(640, 480) shp_filename = os.path.join(CURRENT_DIR, "input", "2015_loaded_NoCC.shp") params = { "shp_filename": shp_filename, "latitude": 40, "longitude": -120, "zoom_start": 5, } url = folium_app.create_url("load_shapefile", params=params) self.view.load(url) def main(): app = QtWidgets.QApplication([]) folium_app.init_handler() w = LeafWidget() w.show() app.exec_() if __name__ == "__main__": main()

Answer 2

So the problem is that my Geojson file is apparently too big. About 34MB with 51k records and folium somehow can't handle it. I tested a smaller json file and it works. So, the question now is what other options do I have to display the map and this big shapefile/json?