Optimizing script

Question

I am working with high resolution raster data on a enormous land cover. I have achieved what I set out to (in terms of script result) and it works very well on a small raster file, however when applying it to a big raster file it takes ages. The work flow is this:

1. Get aspect and slope raster from a DEM raster file
2. Convert the aspect and slope rasters to polygon layers
3. Export certain aspect and slope combinations that are of importance
4. Merge the exported ones into on final layer, which in my case represents "NO-GO-ZONES".

Code:

import pandas as pd
from os.path import join, normpath
import time

start = time.time()

path = 'C:/Users/tlind/Dropbox/Documents/Temp/'

# iface.addRasterLayer(path+'riktigk_raster.tif')

processing.run("native:slope", {'INPUT':path+'riktig.tif','Z_FACTOR':1,'OUTPUT':path+'slope.tif'})
# iface.addRasterLayer(path+'slope.tif')
#
processing.run("native:aspect", {'INPUT':path+'riktig.tif','Z_FACTOR':1,'OUTPUT':path+'aspect.tif'})
# iface.addRasterLayer(path+'aspect.tif')

processing.run("gdal:merge", {'INPUT':['C:/Users/tlind/Dropbox/Documents/Temp/aspect.tif','C:/Users/tlind/Dropbox/Documents/Temp/slope.tif'],'PCT':True,'SEPARATE':True,'NODATA_INPUT':None,'NODATA_OUTPUT':None,'OPTIONS':'','EXTRA':'','DATA_TYPE':5,'OUTPUT':path+'merge.tif'})

processing.run("native:pixelstopolygons", {'INPUT_RASTER':path+'merge.tif','RASTER_BAND':1,'FIELD_NAME':'ASPECT','OUTPUT':path+'aspect.shp'})

processing.run("native:pixelstopolygons", {'INPUT_RASTER':path+'merge.tif','RASTER_BAND':2,'FIELD_NAME':'SLOPE','OUTPUT':path+'slope.shp'})

aspect_layer = iface.addVectorLayer(path+'aspect.shp', "", "ogr")
slope_layer = iface.addVectorLayer(path+'slope.shp', "", "ogr")

pv_apsect = aspect_layer.dataProvider()
pv_apsect.addAttributes([QgsField('ID', QVariant.Double)])

aspect_layer.updateFields()

expression = QgsExpression('$id')

context = QgsExpressionContext()
context.appendScopes(QgsExpressionContextUtils.globalProjectLayerScopes(aspect_layer))

with edit(aspect_layer):
    for f in aspect_layer.getFeatures():
        context.setFeature(f)
        f['ID'] = expression.evaluate(context)
        aspect_layer.updateFeature(f)
    
pv_slope = slope_layer.dataProvider()
pv_slope.addAttributes([QgsField('ID', QVariant.Double)])

slope_layer.updateFields()

expression = QgsExpression('$id')

context = QgsExpressionContext()
context.appendScopes(QgsExpressionContextUtils.globalProjectLayerScopes(slope_layer))

with edit(slope_layer):
    for f in slope_layer.getFeatures():
        context.setFeature(f)
        f['ID'] = expression.evaluate(context)
        slope_layer.updateFeature(f)
    
processing.run("native:joinattributestable", {'INPUT':path+'aspect.shp','FIELD':'ID','INPUT_2':path+'slope.shp','FIELD_2':'ID','FIELDS_TO_COPY':[],'METHOD':1,'DISCARD_NONMATCHING':False,'PREFIX':'','OUTPUT':path+'aspect_slope.shp'})
aspect_slope_layer = iface.addVectorLayer(path+'aspect_slope.shp', "", "ogr")

slope_list = [4.2, 4.6, 5.1, 5.7, 6.4, 7, 9.5, 12, 15, 18.2, 22.5, 30]
aspect_list_1 = [[0, 15], [15,30], [30, 45], [45, 60], [60, 75], [75, 90], [90, 105], [105, 120], [120, 135], [135, 150], [150, 165], [165, 180]]
aspect_list_2 = [[180, 195], [195, 210], [210, 225], [225, 240], [240, 255], [255, 270], [270, 285], [285, 300], [300, 315], [315, 330], [330, 345], [345, 360]]
# aspect_list_3 = aspect_list_1+aspect_list_2

for i in range(len(slope_list)):
    aspect_list_1[i].append(slope_list[i])

for i in range(len(slope_list)):
    aspect_list_2[i].append(slope_list[::-1][i])

for aspect_interval in aspect_list_1:
        start = aspect_interval[0] 
        end = aspect_interval[1]
        slope_loop = aspect_interval[2]
        aspect_slope_layer.selectByExpression('"ASPECT">'+str(start)+' and "ASPECT"<='+str(end)+' and "SLOPE">='+str(slope_loop))
        QgsVectorFileWriter.writeAsVectorFormat(aspect_slope_layer, str(path)+'aspect_slope_'+str(start)+'-'+str(end)+'.shp', "UTF-8", aspect_slope_layer.crs(), "ESRI Shapefile", onlySelected=True)
        # iface.addVectorLayer(str(path)+'aspect_slope_'+str(start)+'-'+str(end)+'.shp', "", "ogr")

for aspect_interval in aspect_list_2:
        start = aspect_interval[0] 
        end = aspect_interval[1]
        slope_loop = aspect_interval[2]
        aspect_slope_layer.selectByExpression('"ASPECT">'+str(start)+' and "ASPECT"<='+str(end)+' and "SLOPE">='+str(slope_loop))
        QgsVectorFileWriter.writeAsVectorFormat(aspect_slope_layer, str(path)+'aspect_slope_'+str(start)+'-'+str(end)+'.shp', "UTF-8", aspect_slope_layer.crs(), "ESRI Shapefile", onlySelected=True)
        # iface.addVectorLayer(str(path)+'aspect_slope_'+str(start)+'-'+str(end)+'.shp', "", "ogr")
    
processing.run("native:mergevectorlayers", {'LAYERS':['C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_0-15.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_105-120.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_120-135.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_135-150.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_15-30.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_150-165.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_165-180.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_180-195.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_195-210.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_210-225.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_225-240.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_240-255.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_255-270.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_270-285.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_285-300.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_30-45.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_300-315.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_315-330.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_330-345.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_345-360.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_45-60.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_60-75.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_75-90.shp','C:/Users/tlind/Dropbox/Documents/Temp/aspect_slope_90-105.shp'],'CRS':None,'OUTPUT':str(path)+'aspect_slope_final.shp'})
iface.addVectorLayer(str(path)+'aspect_slope_final.shp', "", "ogr")

end = time.time()
print("Elapsed time:", (end-start)/60, "minutes.")

Up til creating the polygon layers from the rasters is rather fast. Adding the ID field is something that is rather consuming. Is it possible to produce a vector from raster with adding more than one field from the start, instead of manually having to do it afterwards? In this example. There is the 'FIELD_NAME':'ASPECT'.

processing.run("native:pixelstopolygons", {'INPUT_RASTER':path+'merge.tif','RASTER_BAND':1,'FIELD_NAME':'ASPECT','OUTPUT':path+'aspect.shp'})

When it comes to the actual raster-to-pixel there's not that much to do about it. That takes time.

In the ending loops, where the selected polygons are exported. Is it possible to make a complete selection and do perhaps only one export? That would also speed things up, I guess.

Another thing slightly unlrelated question would be if it's possible to speed things up by making QGIS use more of the CPU? Currently I think it's surfs only on one core.

Answer 1

I managed to optimise it really well!

First replacing all of

pv_apsect = aspect_layer.dataProvider()
pv_apsect.addAttributes([QgsField('ID', QVariant.Double)])

aspect_layer.updateFields()

expression = QgsExpression('$id')

context = QgsExpressionContext()
context.appendScopes(QgsExpressionContextUtils.globalProjectLayerScopes(aspect_layer))

with edit(aspect_layer):
    for f in aspect_layer.getFeatures():
        context.setFeature(f)
        f['ID'] = expression.evaluate(context)
        aspect_layer.updateFeature(f)
    
pv_slope = slope_layer.dataProvider()
pv_slope.addAttributes([QgsField('ID', QVariant.Double)])

slope_layer.updateFields()

expression = QgsExpression('$id')

context = QgsExpressionContext()
context.appendScopes(QgsExpressionContextUtils.globalProjectLayerScopes(slope_layer))

with edit(slope_layer):
    for f in slope_layer.getFeatures():
        context.setFeature(f)
        f['ID'] = expression.evaluate(context)
        slope_layer.updateFeature(f)
    
processing.run("native:joinattributestable", {'INPUT':path+'aspect.shp','FIELD':'ID','INPUT_2':path+'slope.shp','FIELD_2':'ID','FIELDS_TO_COPY':[],'METHOD':1,'DISCARD_NONMATCHING':False,'PREFIX':'','OUTPUT':path+'aspect_slope.shp'})

with simply

processing.run("native:joinattributesbylocation", {'INPUT':path+'slope.shp','PREDICATE':[5],'JOIN':path+'aspect.shp','JOIN_FIELDS':[],'METHOD':0,'DISCARD_NONMATCHING':False,'PREFIX':'','OUTPUT':path+'aspect_slope.shp'})

Additionally I added a spatial index to the shape files with

processing.run("native:createspatialindex", {'INPUT':'LAYER'})

Went from a 28 hour process to a 50 minute process.