THREDDS Crawler Class written in Groovy

Ported from Java:

TdsCrawler.groovy

package util

import java.util.ArrayList
import java.util.List

import javax.naming.Context

import org.apache.log4j.Logger

import thredds.catalog.InvAccess
import thredds.catalog.InvCatalogRef
import thredds.catalog.InvDataset
import thredds.catalog.InvService
import thredds.catalog.ServiceType
import thredds.catalog.crawl.CatalogCrawler

public class TdsCrawler {
 def logger

 private Map urlOpenDAPMap = new HashMap()
 private boolean nestedCrawl = false
 private String topCatalogUrl = ""

 public void setNested(boolean nested) {
  nestedCrawl = nested
 }

 private void doHarvest(InvDataset ids) {

  java.util.List access = ids.getAccess()
  if (access.size() > 0) {

   for (InvAccess a : access) {
    InvService s = a.getService()
    ServiceType stype = s.getServiceType()
    if (stype == ServiceType.OPENDAP) {
     String urlString = a.getStandardUrlName()
     String dsCatalogUrl = ids.getCatalogUrl()
     String datasetId = ids.getID()

     if (nestedCrawl) {
      urlOpenDAPMap.put(datasetId, urlString)
     } else {
      if (dsCatalogUrl.contains(topCatalogUrl)) urlOpenDAPMap.put(datasetId, urlString)
     }
    }
   }
  }
 }

 public void crawl(String catalogUrl, boolean nested) {
  logger = Logger.getLogger(TdsCrawler.class)
  topCatalogUrl = catalogUrl
  nestedCrawl = nested
  Context ctx = null
  final List crawlLst = new ArrayList()
  CatalogCrawler.Listener listener = new CatalogCrawler.Listener() {

     public boolean getCatalogRef(InvCatalogRef catRef, Object obj) {

      if (catRef != null) {
       crawlLst.add(catRef.getURI().toASCIIString())
       return true
      }
      return false
     }

     public void getDataset(InvDataset ids, Object crawler) {
      doHarvest(ids)
     }
    }

  CatalogCrawler crawler = new CatalogCrawler(
    CatalogCrawler.USE_ALL, false, listener)

  crawler.crawl(
    catalogUrl,
    null, System.out, ctx)
 }

 public Map getOpenDAPUrls() {
  return urlOpenDAPMap
 }
}

Test class TdsCrawlerTest.groovy

package util

import static org.junit.Assert.*
import org.junit.Test
import org.apache.log4j.Logger
import services.ServicesMonitor

class TdsCrawlerTest {
 def logger
 
 @Test
 public void crawlTest() {
  logger = Logger.getLogger(TdsCrawlerTest.class)
  def tdsEndPoint = "http://www.ngdc.noaa.gov/thredds/bathyCatalog.xml"

  def tds = new TdsCrawler()
  logger.debug "tdsEndPoint=" + tdsEndPoint
  tds.crawl(tdsEndPoint, false)
  
  def openDapEndPts = tds.getOpenDAPUrls()
  println "odpSize=" + openDapEndPts.size()
  println "Loop thru map:"
  

  
  assertTrue(openDapEndPts.size() > 0)

 }
}

Metadata Mission!

Metadata Mission!

Spock teaches Kirk about the importance of metadata.

Groovy/Grails and Oracle Spatial (Part 1)

Below I document a relatively lightweight approach to perform a distance based search in Oracle Spatial using Groovy and Grails. First some background, we need to calculate reporting metrics for an ocean observing system, one of those metrics is to find all XBT drops within a specified distance of high priority tracklines.

The ultimate goal is to create a web page where a user can specify a trackline and a distance to generate a report on the number of XBT drops along that transect. To begin I'll create a groovy service class to begin testing out my Oracle Spatial Query.

First we specify a database connection (see def serviceDbHandle). You'll also see a createDbHandle method which first checks to see if the db handle has already been made (which is the case when running in grails) otherwise it creates the connection (such as when we are running a unit test of the service in groovy).

Next, our serviceMethod accepts a parameter for sensor type, transect identifier, and a distance. In this example, we are looking for all XBT observations (points) that fall within 2KM of transect AX07. Lastly, we capture are query results as a list of domain objects representing the observations that meet our criteria.

SearchByDistance Service:

//requires Oracle jar in classpath
package osmcreporter

import groovy.sql.Sql

class SearchByDistanceService {

    static transactional = false
 
 def serviceDbHandle = null
 
 def setDbHandle(dataSource) {
   serviceDbHandle = new Sql(dataSource)
 }
 
 //This method allows us to test from Groovy and Grails
 //Lazy init the servicDbHandle
 def createDbHandle () {
  if (serviceDbHandle) return serviceDbHandle
  serviceDbHandle = Sql.newInstance(
   'jdbc:oracle:thin:@myserver:1521:MYINSTANCE',
   'myuser', 'mypasswd', 'oracle.jdbc.driver.OracleDriver')
  return serviceDbHandle
 }
 
    def serviceMethod(sensor_type, transect_id, dist) {
  
  println "\nBegin dsrs.serviceMethod()"
  def DailySummaryObservations = []
  def distance = "distance=$dist"
  createDbHandle()
        def sql = """select ds.platform_id as "platformId", ds.platform_code as "platformCode", ds.parameter_standard_name as "paramStandardName", ds.observation_value as "obsValue", ds.observation_depth as "obsDepth", ds.sensor_type as "sensorType", ds.observation_date as "obsDate", ds.observation_location.sdo_point.y as "obsLat", ds.observation_location.sdo_point.x as "obsLon", ds.platform_type_name as "platformTypeName" from OSMCV4.DS_XBT_2010 ds, OSMCV4.XBT_TRAJECTORY_TSQL xbt where sensor_type = ${sensor_type} AND xbt.line = $transect_id and SDO_WITHIN_DISTANCE(ds.observation_location, xbt.shape, '$distance') = 'TRUE'"""
  serviceDbHandle.eachRow(sql) { 
   DailySummaryObs dso = new DailySummaryObs(it.toRowResult())
   DailySummaryObservations << dso
     }
  println "End dsrs.serviceMethod()"
  return DailySummaryObservations
    } 
 
}

Domain class representing the observations:

package osmcreporter

class DailySummaryObs {

 Long platformId
 String platformCode
 String orgName
 String countryName
 String platformTypeName
 Date obsDate
 String paramStandardName
 String paramUnits
 String sensorCode
 String sensorType
 Integer sensorObsCount
 Integer dailyObsCount
 Double obsDepth
 Double obsValue
 Double obsLat
 Double obsLon 
 
}

Now we'll set up a test class to view our results.

package osmcreporter

import grails.test.*

class SearchByDistanceServiceTests extends GrailsUnitTestCase {
  protected void setUp() {
      super.setUp()
  }

  protected void tearDown() {
      super.tearDown()
  }

    void testSomething() {
        def dsrs = new SearchByDistanceService()
        def List dsos = dsrs.serviceMethod('XBT','AX07','200000') 
 assert dsos.size() >= 0
    }
}

And run the test (provide a program argument - test/unit/osmcreporter/SearchByDistanceServiceTests.groovy):



That's it! We have a working Oracle Spatial query running as a service in groovy whose results populate our domain object that holds the observational data. We also have a unit test case for the service. In our next post, we'll build out the rest of our grails architecture. Namely a GSP page to accept user input, a controller to handle the user's submission, and a barebones GSP page to display the results.

GIS Complex Event Processing Examples

geocommons.com/maps/37577

http://esper.codehaus.org/

References Manning.com - See Event Processing in Action

EsperTech.com

Spring Web MVC registering multiple controllers

I've recently been working on an OGC Sensor Observation Service (SOS) and wanted to split out some of the request mapping for different underlying data sources into separate controllers.

So for example I may have a separate controller for each SOS that is instantiated and which may have different presentations and models to support that view. It wasn't entirely obvious how to 'register' separate controllers into one web app using the Spring servlet config files and web.xml so thought I'd post an example. Note I am using Spring 2.5.x at the time of writing this post and using an annotation based controller.

First, we need to setup our web.xml file to register the Spring dispatcher servlet with the webapp and map which incoming URL requests will be handled by the Spring dispatcher. In the example below I've named the Spring DispatcherServlet sensorObsImpl and registered to incoming URL request /osmcObsService and /nosaObsService to the dispatcher.

Next we need to set-up the WebApplicationContext configuration file for Spring. This file has the same name as your dispatcher servlet with the addition of -servlet.xml extension, so in our case we will have a sensorObsImpl-servlet.xml file. This file sits in the same directory as web.xml. In our file we'll register two controllers one for our OSMC project and one for our NOSA project. Since we are going to leverage Spring's support of annotation based programming we'll begin by enabling auto detection of our annotated controllers by adding component scanning to our config file, followed by defining our to controller beans.

Now our controllers are registered with the dispatcher servlet and we can begin to handle incoming request params with the addition of some annotated methods in our controller classes. In our example below incoming URL requests like /nosaObsService?request=sosInitialize will map to the NosaObsController and be handled by the sosInitialize method.

@Controller
@RequestMapping("/nosaObsService")
public class NosaObsController {

@RequestMapping(params = "request=sosInitialize")
public ModelAndView sosInitialize(@RequestParam("request") String cmd){
.....
}

}

For more information check out the Spring docs at :
http://static.springframework.org/spring/docs/2.5.x/reference/mvc.html

ArcGIS Server REST Admin

You can create a cache for a map service but it will not become available via the REST API until you clear the map cache via the REST admin page.

The REST Admin interface can be accessed via:
http://host:8399/arcgis/rest/admin/

Once the map cache has been cleared to check that the service recognizes it you should see:

Single Fused Map Cache: true

via the rest services directory e.g.
http://host:8399/arcgis/rest/services/myservicename/MapServer

Oracle SDO2FreeEarth Step 3

Lastly we'll reference our javascript file in our HTML page to display our points in FreeEarth.

First reference our javascript file:

script type="text/javascript" src="javascript/myfreearthpts.js"

Next declare variables, and functions

var map;

function randomPtPan() {
var ll = map.getTargetLatLng();
var max = ptlocations.length-1; //normally total number of points
var randomPt = Math.floor(Math.random()*max+1);
ll.lat = ptlocations[randomPt][1];
ll.lng = ptlocations[randomPt][2];

map.panTo(ll, 4, 'easeinquad');

_timeout = setTimeout(randomPtPan, 6000);
}

function onMapLoad() {
// The pts variable is defined in myfreeearthpts.js
//map.zoomTo(90000, 1, 'easeinquad')
for(var i=0; i var code = ptlocations[i][0];
var label = new FE.Label(new FE.LatLng(ptlocations[i][1], ptlocations[i][2]), ptlocations[i][3]);
var icon = new FE.Icon('http://freeearth.poly9.com/images/fff/icons/anchor.png');
var marker = new FE.Pushpin(new FE.LatLng(ptlocations[i][1], ptlocations[i][2]), icon);
this.addOverlay(marker);
}
this.toggleAtmosphere();
setTimeout(randomPtPan, 1);
}

function load() {
map = new FE.Map(document.getElementById("map"));
map.onLoad = onMapLoad;
map.load();
}