[master] bump to GeoServer 2.19.x (GeoNode#8127)

* bump to 2.19.x * [CircleCI] Fix tests * [CircleCI] Fix tests * [CircleCI] Fix tests * fixed remaining dropbox links pointing to artifacts and replaced with correct ones for geoserver 2.19.x * fix for circleci, letsencrypt Dockerfile movced in scripts/docker * [CircleCI] Fix tests * [CircleCI] Fix tests * fix letencrypt new path * [CircleCI] Fix tests * [CircleCI] Fix tests Co-authored-by: Alessio Fabiani <alessio.fabiani@geo-solutions.it>
fvicent · Sep 17, 2021 · 2fbd233 · 2fbd233
1 parent 95f8924
commit 2fbd233
Show file tree

Hide file tree

Showing 41 changed files with 57 additions and 1,828 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,6 +3,8 @@ version: 2.1
 jobs:
   build:
 
+    parallelism: 1
+
     docker:
       - image: circleci/buildpack-deps:focal
 

diff --git a/dev_config.yml b/dev_config.yml
@@ -1,6 +1,6 @@
 ---
-GEOSERVER_URL: "https://www.dropbox.com/s/xlich7pmneaupqp/geoserver-2.18.2.war?dl=1"
-DATA_DIR_URL: "https://www.dropbox.com/s/q0qc2t7d9alo9fk/data-2.18.2.zip?dl=1"
+GEOSERVER_URL: "https://artifacts.geonode.org/geoserver/2.19.x/geoserver.war"
+DATA_DIR_URL: "https://artifacts.geonode.org/geoserver/2.19.x/geonode-geoserver-ext-web-app-data.zip"
 JETTY_RUNNER_URL: "https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.31.v20200723/jetty-runner-9.4.31.v20200723.jar"
 WINDOWS:
   py2exe: "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe"

diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml
@@ -2,7 +2,7 @@ version: '2.2'
 services:
 
   data-dir-conf:
-    image: geonode/geoserver_data:2.18.2
+    image: geonode/geoserver_data:2.19.x
     restart: on-failure
     container_name: gsconf4${COMPOSE_PROJECT_NAME}
     labels:
@@ -13,7 +13,7 @@ services:
       - geoserver-data-dir:/geoserver_data/data
 
   geoserver:
-    image: geonode/geoserver:2.18.2
+    image: geonode/geoserver:2.19.x
     restart: unless-stopped
     container_name: geoserver4${COMPOSE_PROJECT_NAME}
     stdin_open: true

diff --git a/docker-compose-test.yml b/docker-compose-test.yml
@@ -72,7 +72,7 @@ services:
   # Gets and installs letsencrypt certificates
   letsencrypt:
     image: geonode/spcgeonode:letsencrypt-3.1
-    build: ./scripts/spcgeonode/letsencrypt/
+    build: ./scripts/docker/letsencrypt/
     container_name: letsencrypt4${COMPOSE_PROJECT_NAME}
     environment:
       - HTTPS_HOST=${HTTPS_HOST}
@@ -85,7 +85,7 @@ services:
 
   # Geoserver backend
   geoserver:
-    image: geonode/geoserver:2.18.2
+    image: geonode/geoserver:2.19.x
     container_name: geoserver4${COMPOSE_PROJECT_NAME}
     healthcheck:
       test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8080/geoserver/rest/workspaces/geonode.html"
@@ -104,7 +104,7 @@ services:
     restart: on-failure
 
   data-dir-conf:
-    image: geonode/geoserver_data:2.18.2
+    image: geonode/geoserver_data:2.19.x
     container_name: gsconf4${COMPOSE_PROJECT_NAME}
     command: /bin/true
     volumes:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -72,7 +72,7 @@ services:
   # Gets and installs letsencrypt certificates
   letsencrypt:
     image: geonode/spcgeonode:letsencrypt-3.1
-    build: ./scripts/spcgeonode/letsencrypt/
+    build: ./scripts/docker/letsencrypt/
     container_name: letsencrypt4${COMPOSE_PROJECT_NAME}
     environment:
       - HTTPS_HOST=${HTTPS_HOST}
@@ -85,7 +85,7 @@ services:
 
   # Geoserver backend
   geoserver:
-    image: geonode/geoserver:2.18.2
+    image: geonode/geoserver:2.19.x
     container_name: geoserver4${COMPOSE_PROJECT_NAME}
     healthcheck:
       test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8080/geoserver/rest/workspaces/geonode.html"
@@ -104,7 +104,7 @@ services:
     restart: on-failure
 
   data-dir-conf:
-    image: geonode/geoserver_data:2.18.2
+    image: geonode/geoserver_data:2.19.x
     container_name: gsconf4${COMPOSE_PROJECT_NAME}
     command: /bin/true
     volumes:

diff --git a/geonode/geoserver/security.py b/geonode/geoserver/security.py
@@ -229,7 +229,7 @@ def purge_geofence_dataset_rules(resource):
     headers = {'Content-type': 'application/json'}
     workspace = get_dataset_workspace(resource.dataset)
     dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \
-        else resource.dataset.alternate.split(":")[0]
+        else resource.dataset.alternate
     try:
         r = requests.get(
             f"{url}rest/geofence/rules.json?workspace={workspace}&layer={dataset_name}",
@@ -448,7 +448,7 @@ def set_geofence_all(instance):
     logger.debug(f"Inside set_geofence_all for instance {instance}")
     workspace = get_dataset_workspace(resource.dataset)
     dataset_name = resource.dataset.name if resource.dataset and hasattr(resource.dataset, 'name') \
-        else resource.dataset.alternate.split(":")[0]
+        else resource.dataset.alternate
     logger.debug(f"going to work in workspace {workspace}")
     try:
         url = settings.OGC_SERVER['default']['LOCATION']
@@ -493,7 +493,7 @@ def sync_geofence_with_guardian(dataset, perms, user=None, group=None, group_per
     """
     Sync Guardian permissions to GeoFence.
     """
-    _dataset_name = dataset.name if dataset and hasattr(dataset, 'name') else dataset.alternate.split(":")[0]
+    _dataset_name = dataset.name if dataset and hasattr(dataset, 'name') else dataset.alternate
     _dataset_workspace = get_dataset_workspace(dataset)
     # Create new rule-set
     gf_services = _get_gf_services(dataset, perms)

diff --git a/geonode/geoserver/tests/test_manager.py b/geonode/geoserver/tests/test_manager.py
@@ -17,6 +17,7 @@
 #
 #########################################################################
 import os
+import base64
 import shutil
 import gisdata
 import requests
@@ -57,15 +58,21 @@ def tearDown(self) -> None:
     def test_revise_resource_value_in_append_should_add_expected_rows_in_the_catalog(self):
         layer = Dataset.objects.get(name=self.sut.name)
         _gs_import_session_info = self.geoserver_manager._revise_resource_value(layer, list(self.files_as_dict.values()), self.user, action_type="append")
-        result = requests.get(f'{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}')
+        basic_auth = base64.b64encode(b'admin:geoserver')
+        result = requests.get(
+            f'{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}',
+            headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"})
         self.assertEqual(result.status_code, 200)
         self.assertEqual(result.json().get('import').get('state'), enumerations.STATE_COMPLETE)
 
     @on_ogc_backend(geoserver.BACKEND_PACKAGE)
     def test_revise_resource_value_in_replace_should_add_expected_rows_in_the_catalog(self):
         layer = Dataset.objects.get(name=self.sut.name)
         _gs_import_session_info = self.geoserver_manager._revise_resource_value(layer, list(self.files_as_dict.values()), self.user, action_type="replace")
-        result = requests.get(f'{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}')
+        basic_auth = base64.b64encode(b'admin:geoserver')
+        result = requests.get(
+            f'{self.geoserver_url}/rest/imports/{_gs_import_session_info.import_session.id}',
+            headers={"Authorization": f"Basic {basic_auth.decode('utf-8')}"})
         self.assertEqual(result.status_code, 200)
         self.assertEqual(result.json().get('import').get('state'), enumerations.STATE_COMPLETE)
 

diff --git a/geonode/geoserver/views.py b/geonode/geoserver/views.py
@@ -543,7 +543,10 @@ def _response_callback(**kwargs):
         # Replace Proxy URL
         try:
             if isinstance(content, bytes):
-                _content = content.decode('UTF-8')
+                try:
+                    _content = content.decode('UTF-8')
+                except UnicodeDecodeError:
+                    _content = content
             else:
                 _content = content
             if re.findall(f"(?=(\\b{'|'.join(content_type_list)}\\b))", content_type):

diff --git a/geonode/security/tests.py b/geonode/security/tests.py
@@ -23,7 +23,6 @@
 import importlib
 
 from requests.auth import HTTPBasicAuth
-from urllib.request import urlopen, Request
 from tastypie.test import ResourceTestCaseMixin
 
 from django.db.models import Q
@@ -47,7 +46,6 @@
 from geonode.tests.utils import check_dataset
 from geonode.decorators import on_ogc_backend
 from geonode.geoserver.helpers import gs_slurp
-from geonode.people.utils import get_valid_user
 from geonode.resource.manager import resource_manager
 from geonode.tests.base import GeoNodeBaseTestSupport
 from geonode.groups.models import Group, GroupProfile
@@ -667,7 +665,7 @@ def test_perm_specs_synchronization(self):
 
                 self.assertTrue('limits' in rule)
                 rule_limits = rule['limits']
-                self.assertEqual(rule_limits['allowedArea'], 'MULTIPOLYGON (((145.8046418749977 -42.49606500060302, \
+                self.assertEqual(rule_limits['allowedArea'], 'SRID=4326;MULTIPOLYGON (((145.8046418749977 -42.49606500060302, \
 146.7000276171853 -42.53655428642583, 146.7110139453067 -43.07256577359489, \
 145.9804231249952 -43.05651288026286, 145.8046418749977 -42.49606500060302)))')
                 self.assertEqual(rule_limits['catalogMode'], 'MIXED')
@@ -707,7 +705,7 @@ def test_perm_specs_synchronization(self):
 
                     self.assertTrue('limits' in rule)
                     rule_limits = rule['limits']
-                    self.assertEqual(rule_limits['allowedArea'], 'MULTIPOLYGON (((145.8046418749977 -42.49606500060302, \
+                    self.assertEqual(rule_limits['allowedArea'], 'SRID=4326;MULTIPOLYGON (((145.8046418749977 -42.49606500060302, \
 146.7000276171853 -42.53655428642583, 146.7110139453067 -43.07256577359489, \
 145.9804231249952 -43.05651288026286, 145.8046418749977 -42.49606500060302)))')
                     self.assertEqual(rule_limits['catalogMode'], 'MIXED')
@@ -743,7 +741,7 @@ def test_perm_specs_synchronization(self):
                     self.assertTrue('limits' in rule)
                     rule_limits = rule['limits']
                     self.assertEqual(
-                        rule_limits['allowedArea'], 'MULTIPOLYGON (((145.8046418749977 -42.49606500060302, 146.7000276171853 \
+                        rule_limits['allowedArea'], 'SRID=4326;MULTIPOLYGON (((145.8046418749977 -42.49606500060302, 146.7000276171853 \
 -42.53655428642583, 146.7110139453067 -43.07256577359489, 145.9804231249952 \
 -43.05651288026286, 145.8046418749977 -42.49606500060302)))')
                     self.assertEqual(rule_limits['catalogMode'], 'MIXED')
@@ -979,59 +977,39 @@ def test_dataset_upload_with_time(self):
     def test_dataset_permissions(self):
         # Test permissions on a layer
         bobby = get_user_model().objects.get(username="bobby")
-        layer = create_single_dataset('san_andres_y_providencia_poi.shp')
+        layer = create_single_dataset('san_andres_y_providencia_poi')
         layer = resource_manager.update(
             layer.uuid,
             instance=layer,
             notify=False,
             vals=dict(
-                owner=bobby
+                owner=bobby,
+                workspace=settings.DEFAULT_WORKSPACE
             ))
 
         self.assertIsNotNone(layer)
         self.assertIsNotNone(layer.ows_url)
         self.assertIsNotNone(layer.ptype)
         self.assertIsNotNone(layer.sourcetype)
+        self.assertEqual(layer.alternate, 'geonode:san_andres_y_providencia_poi')
 
         # Reset GeoFence Rules
         purge_geofence_all()
         geofence_rules_count = get_geofence_rules_count()
-        self.assertTrue(geofence_rules_count == 0)
-
-        ignore_errors = True
-        skip_unadvertised = False
-        skip_geonode_registered = False
-        remove_deleted = True
-        verbosity = 2
-        owner = get_valid_user('admin')
-        workspace = 'geonode'
-        filter = None
-        store = None
-        permissions = {'users': {"admin": ['change_dataset_data']}, 'groups': []}
-        gs_slurp(
-            ignore_errors=ignore_errors,
-            verbosity=verbosity,
-            owner=owner,
-            console=StreamToLogger(logger, logging.INFO),
-            workspace=workspace,
-            store=store,
-            filter=filter,
-            skip_unadvertised=skip_unadvertised,
-            skip_geonode_registered=skip_geonode_registered,
-            remove_deleted=remove_deleted,
-            permissions=permissions,
-            execute_signals=True)
+        self.assertEqual(geofence_rules_count, 0)
 
-        layer = Dataset.objects.get(name='san_andres_y_providencia_poi.shp')
+        layer = Dataset.objects.get(name='san_andres_y_providencia_poi')
+        layer.set_default_permissions(owner=bobby)
         check_dataset(layer)
         geofence_rules_count = get_geofence_rules_count()
         _log(f"0. geofence_rules_count: {geofence_rules_count} ")
-        self.assertTrue(geofence_rules_count >= 2)
+        self.assertGreaterEqual(geofence_rules_count, 4)
 
         # Set the layer private for not authenticated users
-        layer.set_permissions({'users': {'AnonymousUser': []}, 'groups': []})
+        perm_spec = {'users': {'AnonymousUser': []}, 'groups': []}
+        layer.set_permissions(perm_spec)
 
-        url = f'{settings.GEOSERVER_LOCATION}geonode/ows?' \
+        url = f'{settings.SITEURL}gs/ows?' \
             'LAYERS=geonode%3Asan_andres_y_providencia_poi&STYLES=' \
             '&FORMAT=image%2Fpng&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap' \
             '&SRS=EPSG%3A4326' \
@@ -1040,44 +1018,28 @@ def test_dataset_permissions(self):
             '&WIDTH=217&HEIGHT=512'
 
         # test view_resourcebase permission on anonymous user
-        request = Request(url)
-        response = urlopen(request)
-        _content_type = response.getheader('Content-Type').lower()
+        response = requests.get(url)
+        self.assertTrue(response.status_code, 404)
         self.assertEqual(
-            _content_type,
-            'application/vnd.ogc.se_xml;charset=utf-8'
+            response.headers.get('Content-Type'),
+            'application/vnd.ogc.se_xml;charset=UTF-8'
         )
 
-        # test WMS with authenticated user that has not view_resourcebase:
-        # the layer must be not accessible (response is xml)
-        request = Request(url)
-        basic_auth = base64.b64encode(b'bobby:bob')
-        request.add_header("Authorization", f"Basic {basic_auth.decode('utf-8')}")
-        response = urlopen(request)
-        _content_type = response.getheader('Content-Type').lower()
+        # test WMS with authenticated user that has access to the Layer
+        response = requests.get(url, auth=HTTPBasicAuth(username=settings.OGC_SERVER['default']['USER'], password=settings.OGC_SERVER['default']['PASSWORD']))
+        self.assertTrue(response.status_code, 200)
         self.assertEqual(
-            _content_type,
-            'application/vnd.ogc.se_xml;charset=utf-8'
+            response.headers.get('Content-Type'),
+            'image/png'
         )
 
-        # test WMS with authenticated user that has view_resourcebase: the layer
-        # must be accessible (response is image)
-        perm_spec = {
-            'users': {
-                'bobby': ['view_resourcebase',
-                          'download_resourcebase']
-            },
-            'groups': []
-        }
-        layer.set_permissions(perm_spec)
-        request = Request(url)
-        basic_auth = base64.b64encode(b'bobby:bob')
-        request.add_header("Authorization", f"Basic {basic_auth.decode('utf-8')}")
-        response = urlopen(request)
-        _content_type = response.getheader('Content-Type').lower()
+        # test WMS with authenticated user that has no view_resourcebase:
+        # the layer should be not accessible
+        response = requests.get(url, auth=HTTPBasicAuth(username='norman', password='norman'))
+        self.assertTrue(response.status_code, 404)
         self.assertEqual(
-            _content_type,
-            'application/vnd.ogc.se_xml;charset=utf-8'
+            response.headers.get('Content-Type'),
+            'text/html;charset=utf-8'
         )
 
         # test change_dataset_style

diff --git a/scripts/spcgeonode/letsencrypt/Dockerfile → scripts/docker/letsencrypt/Dockerfile b/scripts/spcgeonode/letsencrypt/Dockerfile → scripts/docker/letsencrypt/Dockerfile
diff --git a/scripts/spcgeonode/letsencrypt/README.md → scripts/docker/letsencrypt/README.md b/scripts/spcgeonode/letsencrypt/README.md → scripts/docker/letsencrypt/README.md
diff --git a/scripts/spcgeonode/letsencrypt/crontab → scripts/docker/letsencrypt/crontab b/scripts/spcgeonode/letsencrypt/crontab → scripts/docker/letsencrypt/crontab
diff --git a/...cgeonode/letsencrypt/docker-entrypoint.sh → ...s/docker/letsencrypt/docker-entrypoint.sh b/...cgeonode/letsencrypt/docker-entrypoint.sh → ...s/docker/letsencrypt/docker-entrypoint.sh
diff --git a/scripts/spcgeonode/.dockerignore b/scripts/spcgeonode/.dockerignore