concept query in category spark

appears as: query, The query, query
Spark in Action, Second Edition

This is an excerpt from Manning's book Spark in Action, Second Edition.

Listing 8.5 Desired output of the query with a WHERE clause
+-------+--------------+--------------------+------------+-----------+...
|film_id| title        | description        |release_year|language_id|...
+-------+--------------+--------------------+------------+-----------+...
| 6     | AGENT TRUMAN |A Intrepid Panora...| 2005-12-31 | 1         |...
| 13    | ALI FOREVER  |A Action-Packed D...| 2005-12-31 | 1         |...
...
+-------+--------------+--------------------+------------+-----------+...
only showing top 5 rows
 
root
  |-- film_id: integer (nullable = true)
  |-- title: string (nullable = true)
  |-- description: string (nullable = true)
  |-- release_year: date (nullable = true)
  |-- language_id: integer (nullable = true)
  |-- original_language_id: integer (nullable = true)
  |-- rental_duration: integer (nullable = true)
  |-- rental_rate: decimal(4,2) (nullable = true)
  |-- length: integer (nullable = true)
  |-- replacement_cost: decimal(5,2) (nullable = true)
  |-- rating: string (nullable = true)
  |-- special_features: string (nullable = true)
  |-- last_update: timestamp (nullable = true)
 
 The dataframe contains 16 record(s).
Listing 10.6 ReadLinesFromFileStreamApp.java
package net.jgp.books.spark.ch10.lab200_read_stream;
 
import java.util.concurrent.TimeoutException;
 
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.streaming.OutputMode;
import org.apache.spark.sql.streaming.StreamingQuery;
import org.apache.spark.sql.streaming.StreamingQueryException;
import org.slf4j.Logger; #1
import org.slf4j.LoggerFactory; #1
 
import net.jgp.books.spark.ch10.x.utils.streaming.lib.StreamingUtils;
 
public class ReadLinesFromFileStreamApp {
     private static transient Logger log = LoggerFactory.getLogger(
        ReadLinesFromFileStreamApp. class ); #2
 
     public static void main(String[] args ) {
      ReadRecordFromFileStreamApp app = new ReadRecordFromFileStreamApp();
       try {
         app .start();
      } catch (TimeoutException e ) {
         log .error( "A timeout exception has occured: {}" , e .getMessage());
      }
    }
 
     private void start() {
       log .debug( "-> start()" ); #3
 
      SparkSession spark = SparkSession.builder() #4
            .appName( "Read lines over a file stream" ) #4
            .master( "local" ) #4
            .getOrCreate(); #4
 
      Dataset<Row> df = spark
            .readStream() #5
            .format( "text" ) #6
            .load(StreamingUtils.getInputDirectory()); #7
 
      StreamingQuery query = df
            .writeStream() #8
            .outputMode(OutputMode.Append()) #9
            .format( "console" ) #10
            .option( "truncate" , false ) #11
            .option( "numRows" , 3) #11
            .start(); #12
 
       try {
         query .awaitTermination(60000); #13
      } catch (StreamingQueryException e ) {
         log .error(
               "Exception while waiting for query to end {}." ,
               e .getMessage(),
               e );
      }
 
       log .debug( "<- start()" ); #14
    }
 }
  • To wait for the data to come, you can use awaitTermination() , which is a blocking method, or check whether the query is active by using the isActive() method.