concept query in category spark
appears as: query, The query, query

This is an excerpt from Manning's book Spark in Action, Second Edition.
Listing 8.5 Desired output of the query with a WHERE clause
+-------+--------------+--------------------+------------+-----------+... |film_id| title | description |release_year|language_id|... +-------+--------------+--------------------+------------+-----------+... | 6 | AGENT TRUMAN |A Intrepid Panora...| 2005-12-31 | 1 |... | 13 | ALI FOREVER |A Action-Packed D...| 2005-12-31 | 1 |... ... +-------+--------------+--------------------+------------+-----------+... only showing top 5 rows root |-- film_id: integer (nullable = true) |-- title: string (nullable = true) |-- description: string (nullable = true) |-- release_year: date (nullable = true) |-- language_id: integer (nullable = true) |-- original_language_id: integer (nullable = true) |-- rental_duration: integer (nullable = true) |-- rental_rate: decimal(4,2) (nullable = true) |-- length: integer (nullable = true) |-- replacement_cost: decimal(5,2) (nullable = true) |-- rating: string (nullable = true) |-- special_features: string (nullable = true) |-- last_update: timestamp (nullable = true) The dataframe contains 16 record(s).
Listing 10.6 ReadLinesFromFileStreamApp.java
package net.jgp.books.spark.ch10.lab200_read_stream; import java.util.concurrent.TimeoutException; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.streaming.OutputMode; import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.streaming.StreamingQueryException; import org.slf4j.Logger; #1 import org.slf4j.LoggerFactory; #1 import net.jgp.books.spark.ch10.x.utils.streaming.lib.StreamingUtils; public class ReadLinesFromFileStreamApp { private static transient Logger log = LoggerFactory.getLogger( ReadLinesFromFileStreamApp. class ); #2 public static void main(String[] args ) { ReadRecordFromFileStreamApp app = new ReadRecordFromFileStreamApp(); try { app .start(); } catch (TimeoutException e ) { log .error( "A timeout exception has occured: {}" , e .getMessage()); } } private void start() { log .debug( "-> start()" ); #3 SparkSession spark = SparkSession.builder() #4 .appName( "Read lines over a file stream" ) #4 .master( "local" ) #4 .getOrCreate(); #4 Dataset<Row> df = spark .readStream() #5 .format( "text" ) #6 .load(StreamingUtils.getInputDirectory()); #7 StreamingQuery query = df .writeStream() #8 .outputMode(OutputMode.Append()) #9 .format( "console" ) #10 .option( "truncate" , false ) #11 .option( "numRows" , 3) #11 .start(); #12 try { query .awaitTermination(60000); #13 } catch (StreamingQueryException e ) { log .error( "Exception while waiting for query to end {}." , e .getMessage(), e ); } log .debug( "<- start()" ); #14 } }
To wait for the data to come, you can use awaitTermination(), which is a blocking method, or check whether the query is active by using theisActive()method.