From 603b08046a274da4204bf5eed2c9d484430eda2b Mon Sep 17 00:00:00 2001 From: Bill Chambers Date: Tue, 24 Mar 2015 20:24:51 -0700 Subject: [PATCH 01/16] [DOCUMENTATION]Fixed Missing Type Import in Documentation Needed to import the types specifically, not the general pyspark.sql --- docs/sql-programming-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 6a333fdb562a..b8fbe1c723d6 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -624,7 +624,7 @@ tuples or lists in the RDD created in the step 1. For example: {% highlight python %} # Import SQLContext and data types -from pyspark.sql import * +from pyspark.sql.types import * # sc is an existing SparkContext. sqlContext = SQLContext(sc) From 8fa67bf36f8f13e462f45f298c94f9cb3fc6fc1c Mon Sep 17 00:00:00 2001 From: anabranch Date: Tue, 24 Mar 2015 20:48:45 -0700 Subject: [PATCH 02/16] Corrected SqlContext Import --- docs/sql-programming-guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index b8fbe1c723d6..c99a0b03442c 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -624,6 +624,7 @@ tuples or lists in the RDD created in the step 1. For example: {% highlight python %} # Import SQLContext and data types +from pyspark.sql import SQLContext from pyspark.sql.types import * # sc is an existing SparkContext. From 0c391d9c1705d48ca2a16091d12a72997b189b8a Mon Sep 17 00:00:00 2001 From: anabranch Date: Mon, 7 Dec 2015 13:00:26 -0800 Subject: [PATCH 03/16] [SPARK-11964]pipeline persistence=>ml-guide --- docs/ml-guide.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 6f35b30c3d4d..b7a15b0ab447 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -951,4 +951,77 @@ model.transform(test) {% endhighlight %} - \ No newline at end of file + + +## Example: Saving and Loading a Previously Created Model Pipeline + +Often times it is worth it to save a model to disk for usage later. In Spark 1.6, similar model import/export functionality was added to the Pipeline API. All basic transformers are now supports as well as several Spark ML methods. Below is a list of the currently supported models. + +* AFTSurvivalRegression +* IsotonicRegression +* LDA +* K-Means +* Naive Bayes +* ALS +* Linear Regression +* Logistic Regression +* Decision Tree Classifiers and Regressors +* RFormulas +* Multilayer Perceptrons + + +Below is an example of how a pipeline can be persisted and loaded. We are using one of the pipelines from the previous example to do so. +
+{% highlight scala %} +import org.apache.spark.ml.Pipeline +import org.apache.spark.ml.classification.LogisticRegression +import org.apache.spark.ml.feature.{HashingTF, Tokenizer} +import org.apache.spark.mllib.linalg.Vector +import org.apache.spark.sql.Row + +// Prepare training documents from a list of (id, text, label) tuples. +val training = sqlContext.createDataFrame(Seq( +(0L, "a b c d e spark", 1.0), +(1L, "b d", 0.0), +(2L, "spark f g h", 1.0), +(3L, "hadoop mapreduce", 0.0) +)).toDF("id", "text", "label") + +// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr. +val tokenizer = new Tokenizer() +.setInputCol("text") +.setOutputCol("words") +val hashingTF = new HashingTF() +.setNumFeatures(1000) +.setInputCol(tokenizer.getOutputCol) +.setOutputCol("features") +val lr = new LogisticRegression() +.setMaxIter(10) +.setRegParam(0.01) +val pipeline = new Pipeline() +.setStages(Array(tokenizer, hashingTF, lr)) + +val model = pipeline.fit(training) +model.save("/tmp/spark-logistic-regression-model") + +// to load in the model +val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") +// or equivalently +Pipeline.read.load("/tmp/spark-logistic-regression-model") +val test = sqlContext.createDataFrame(Seq( +(4L, "spark i j k"), +(5L, "l m n"), +(6L, "mapreduce spark"), +(7L, "apache hadoop") +)).toDF("id", "text") + +// Make predictions on test documents. +loadedModel.transform(test) +.select("id", "text", "probability", "prediction") +.collect() +.foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => +println(s"($id, $text) --> prob=$prob, prediction=$prediction") + } +{% endhighlight %} +
+ From a6809bdefc581dbb1f4cac01c1c6c4b97951ed7f Mon Sep 17 00:00:00 2001 From: anabranch Date: Mon, 7 Dec 2015 13:13:08 -0800 Subject: [PATCH 04/16] improved formatting, description --- docs/ml-guide.md | 58 ++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index b7a15b0ab447..a386f5215e2d 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -955,22 +955,15 @@ model.transform(test) ## Example: Saving and Loading a Previously Created Model Pipeline -Often times it is worth it to save a model to disk for usage later. In Spark 1.6, similar model import/export functionality was added to the Pipeline API. All basic transformers are now supports as well as several Spark ML methods. Below is a list of the currently supported models. +Often times it is worth it to save a model to disk for usage later. In Spark 1.6, similar model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML Models such as: -* AFTSurvivalRegression -* IsotonicRegression -* LDA * K-Means * Naive Bayes * ALS * Linear Regression * Logistic Regression -* Decision Tree Classifiers and Regressors -* RFormulas -* Multilayer Perceptrons - -Below is an example of how a pipeline can be persisted and loaded. We are using one of the pipelines from the previous example to do so. +Below is an example of how a pipeline can be persisted and loaded. This example uses a model that we trained and created above.
{% highlight scala %} import org.apache.spark.ml.Pipeline @@ -981,47 +974,48 @@ import org.apache.spark.sql.Row // Prepare training documents from a list of (id, text, label) tuples. val training = sqlContext.createDataFrame(Seq( -(0L, "a b c d e spark", 1.0), -(1L, "b d", 0.0), -(2L, "spark f g h", 1.0), -(3L, "hadoop mapreduce", 0.0) + (0L, "a b c d e spark", 1.0), + (1L, "b d", 0.0), + (2L, "spark f g h", 1.0), + (3L, "hadoop mapreduce", 0.0) )).toDF("id", "text", "label") // Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr. val tokenizer = new Tokenizer() -.setInputCol("text") -.setOutputCol("words") + .setInputCol("text") + .setOutputCol("words") val hashingTF = new HashingTF() -.setNumFeatures(1000) -.setInputCol(tokenizer.getOutputCol) -.setOutputCol("features") + .setNumFeatures(1000) + .setInputCol(tokenizer.getOutputCol) + .setOutputCol("features") val lr = new LogisticRegression() -.setMaxIter(10) -.setRegParam(0.01) + .setMaxIter(10) + .setRegParam(0.01) + val pipeline = new Pipeline() -.setStages(Array(tokenizer, hashingTF, lr)) + .setStages(Array(tokenizer, hashingTF, lr)) val model = pipeline.fit(training) model.save("/tmp/spark-logistic-regression-model") -// to load in the model +// load in the model val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") // or equivalently -Pipeline.read.load("/tmp/spark-logistic-regression-model") +val loadedModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") + val test = sqlContext.createDataFrame(Seq( -(4L, "spark i j k"), -(5L, "l m n"), -(6L, "mapreduce spark"), -(7L, "apache hadoop") + (4L, "spark i j k"), + (5L, "l m n"), + (6L, "mapreduce spark"), + (7L, "apache hadoop") )).toDF("id", "text") // Make predictions on test documents. loadedModel.transform(test) -.select("id", "text", "probability", "prediction") -.collect() -.foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => -println(s"($id, $text) --> prob=$prob, prediction=$prediction") + .select("id", "text", "probability", "prediction") + .collect() + .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => + println(s"($id, $text) --> prob=$prob, prediction=$prediction") } {% endhighlight %}
- From eb3f99c93d6a91d1d6da1765dbdc96d64ab3bf13 Mon Sep 17 00:00:00 2001 From: anabranch Date: Mon, 7 Dec 2015 18:47:40 -0800 Subject: [PATCH 05/16] re-organization of docs + feedback --- docs/ml-guide.md | 109 ++++++++++++++++++----------------------------- 1 file changed, 42 insertions(+), 67 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index a386f5215e2d..8d5d1d5b18ca 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -613,7 +613,49 @@ for row in selected.collect(): {% endhighlight %} + + +## Example: Saving and Loading a Pipeline + +Often times it is worth it to save a model to disk for later use. In Spark 1.6, model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: + +* K-Means +* Naive Bayes +* ALS +* Linear Regression +* Logistic Regression + +Below is an example of how a pipeline can be persisted and loaded. This example uses a model that we trained and created above. +
+ +
+{% highlight scala %} +// fit the model as we did in the previous example +val model = pipeline.fit(training) +// now save it to disk +model.save("/tmp/spark-logistic-regression-model") +// load in the model +val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") +// or equivalently +val loadedModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") + +val test = sqlContext.createDataFrame(Seq( + (4L, "spark i j k"), + (5L, "l m n"), + (6L, "mapreduce spark"), + (7L, "apache hadoop") +)).toDF("id", "text") + +// Make predictions on test documents +loadedModel.transform(test) + .select("id", "text", "probability", "prediction") + .collect() + .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => + println(s"($id, $text) --> prob=$prob, prediction=$prediction") + } +{% endhighlight %} +
## Example: model selection via cross-validation @@ -952,70 +994,3 @@ model.transform(test) - -## Example: Saving and Loading a Previously Created Model Pipeline - -Often times it is worth it to save a model to disk for usage later. In Spark 1.6, similar model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML Models such as: - -* K-Means -* Naive Bayes -* ALS -* Linear Regression -* Logistic Regression - -Below is an example of how a pipeline can be persisted and loaded. This example uses a model that we trained and created above. -
-{% highlight scala %} -import org.apache.spark.ml.Pipeline -import org.apache.spark.ml.classification.LogisticRegression -import org.apache.spark.ml.feature.{HashingTF, Tokenizer} -import org.apache.spark.mllib.linalg.Vector -import org.apache.spark.sql.Row - -// Prepare training documents from a list of (id, text, label) tuples. -val training = sqlContext.createDataFrame(Seq( - (0L, "a b c d e spark", 1.0), - (1L, "b d", 0.0), - (2L, "spark f g h", 1.0), - (3L, "hadoop mapreduce", 0.0) -)).toDF("id", "text", "label") - -// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr. -val tokenizer = new Tokenizer() - .setInputCol("text") - .setOutputCol("words") -val hashingTF = new HashingTF() - .setNumFeatures(1000) - .setInputCol(tokenizer.getOutputCol) - .setOutputCol("features") -val lr = new LogisticRegression() - .setMaxIter(10) - .setRegParam(0.01) - -val pipeline = new Pipeline() - .setStages(Array(tokenizer, hashingTF, lr)) - -val model = pipeline.fit(training) -model.save("/tmp/spark-logistic-regression-model") - -// load in the model -val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") -// or equivalently -val loadedModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") - -val test = sqlContext.createDataFrame(Seq( - (4L, "spark i j k"), - (5L, "l m n"), - (6L, "mapreduce spark"), - (7L, "apache hadoop") -)).toDF("id", "text") - -// Make predictions on test documents. -loadedModel.transform(test) - .select("id", "text", "probability", "prediction") - .collect() - .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => - println(s"($id, $text) --> prob=$prob, prediction=$prediction") - } -{% endhighlight %} -
From bde19ad05b8e85c50f48f6a016e17f3ab8da658e Mon Sep 17 00:00:00 2001 From: anabranch Date: Wed, 9 Dec 2015 23:04:14 -0800 Subject: [PATCH 06/16] feedback from @benFradet --- docs/ml-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 8d5d1d5b18ca..6494e3aa137b 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -617,7 +617,7 @@ for row in selected.collect(): ## Example: Saving and Loading a Pipeline -Often times it is worth it to save a model to disk for later use. In Spark 1.6, model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: +Often times it is worth it to save a model to disk for later use. In Spark 1.6, a model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: * K-Means * Naive Bayes From e1f27eb69a77e53e3080636aef92dc3274ac4ca8 Mon Sep 17 00:00:00 2001 From: anabranch Date: Thu, 10 Dec 2015 21:24:48 -0800 Subject: [PATCH 07/16] update to the formatting --- docs/ml-guide.md | 75 +++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 6494e3aa137b..e7d791bb146f 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -153,8 +153,8 @@ If the `Pipeline` had more stages, it would call the `LogisticRegressionModel`'s method on the `DataFrame` before passing the `DataFrame` to the next stage. A `Pipeline` is an `Estimator`. -Thus, after a `Pipeline`'s `fit()` method runs, it produces a `PipelineModel`, which is a -`Transformer`. +Thus, after a `Pipeline`'s `fit()` method runs, it produces a `PipelineModel`, which is a `Transformer`. + This `PipelineModel` is used at *test time*; the figure below illustrates this usage.

@@ -187,6 +187,15 @@ This type checking is done using the `DataFrame` *schema*, a description of the unique IDs. However, different instances `myHashingTF1` and `myHashingTF2` (both of type `HashingTF`) can be put into the same `Pipeline` since different instances will be created with different IDs. +### Saving Pipelines +Often times it is worth it to save a model or a pipeline to disk for later use. In Spark 1.6, a model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: + +* K-Means +* Naive Bayes +* ALS +* Linear Regression +* Logistic Regression + ## Parameters Spark ML `Estimator`s and `Transformer`s use a uniform API for specifying parameters. @@ -468,6 +477,15 @@ val pipeline = new Pipeline() // Fit the pipeline to training documents. val model = pipeline.fit(training) +// now we can optionally save it to disk +model.save("/tmp/spark-logistic-regression-model") + +// and load it back in during production +val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") +// or equivalently +val loadedModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") + + // Prepare test documents, which are unlabeled (id, text) tuples. val test = sqlContext.createDataFrame(Seq( (4L, "spark i j k"), @@ -484,6 +502,15 @@ model.transform(test) println(s"($id, $text) --> prob=$prob, prediction=$prediction") } +// or use the "loadedModel" to make predictions +// Make predictions on test documents +loadedModel.transform(test) + .select("id", "text", "probability", "prediction") + .collect() + .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => + println(s"($id, $text) --> prob=$prob, prediction=$prediction") + } + {% endhighlight %} @@ -556,6 +583,7 @@ Pipeline pipeline = new Pipeline() // Fit the pipeline to training documents. PipelineModel model = pipeline.fit(training); + // Prepare test documents, which are unlabeled. DataFrame test = sqlContext.createDataFrame(Arrays.asList( new Document(4L, "spark i j k"), @@ -615,49 +643,6 @@ for row in selected.collect(): -## Example: Saving and Loading a Pipeline - -Often times it is worth it to save a model to disk for later use. In Spark 1.6, a model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: - -* K-Means -* Naive Bayes -* ALS -* Linear Regression -* Logistic Regression - -Below is an example of how a pipeline can be persisted and loaded. This example uses a model that we trained and created above. -

- -
-{% highlight scala %} -// fit the model as we did in the previous example -val model = pipeline.fit(training) -// now save it to disk -model.save("/tmp/spark-logistic-regression-model") - -// load in the model -val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") -// or equivalently -val loadedModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") - -val test = sqlContext.createDataFrame(Seq( - (4L, "spark i j k"), - (5L, "l m n"), - (6L, "mapreduce spark"), - (7L, "apache hadoop") -)).toDF("id", "text") - -// Make predictions on test documents -loadedModel.transform(test) - .select("id", "text", "probability", "prediction") - .collect() - .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => - println(s"($id, $text) --> prob=$prob, prediction=$prediction") - } -{% endhighlight %} -
-
- ## Example: model selection via cross-validation An important task in ML is *model selection*, or using data to find the best model or parameters for a given task. This is also called *tuning*. From 4a3b513bb92966bf99e1a291748d4b8668275798 Mon Sep 17 00:00:00 2001 From: anabranch Date: Thu, 10 Dec 2015 21:31:52 -0800 Subject: [PATCH 08/16] updated formatting, better location --- docs/ml-guide.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index e7d791bb146f..1871de375841 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -187,15 +187,6 @@ This type checking is done using the `DataFrame` *schema*, a description of the unique IDs. However, different instances `myHashingTF1` and `myHashingTF2` (both of type `HashingTF`) can be put into the same `Pipeline` since different instances will be created with different IDs. -### Saving Pipelines -Often times it is worth it to save a model or a pipeline to disk for later use. In Spark 1.6, a model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: - -* K-Means -* Naive Bayes -* ALS -* Linear Regression -* Logistic Regression - ## Parameters Spark ML `Estimator`s and `Transformer`s use a uniform API for specifying parameters. @@ -214,6 +205,16 @@ Parameters belong to specific instances of `Estimator`s and `Transformer`s. For example, if we have two `LogisticRegression` instances `lr1` and `lr2`, then we can build a `ParamMap` with both `maxIter` parameters specified: `ParamMap(lr1.maxIter -> 10, lr2.maxIter -> 20)`. This is useful if there are two algorithms with the `maxIter` parameter in a `Pipeline`. +## Saving and Loading Pipelines + +Often times it is worth it to save a model or a pipeline to disk for later use. In Spark 1.6, a model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: + +* K-Means +* Naive Bayes +* ALS +* Linear Regression +* Logistic Regression + # Code examples This section gives code examples illustrating the functionality discussed above. @@ -485,7 +486,6 @@ val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") // or equivalently val loadedModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") - // Prepare test documents, which are unlabeled (id, text) tuples. val test = sqlContext.createDataFrame(Seq( (4L, "spark i j k"), @@ -503,13 +503,12 @@ model.transform(test) } // or use the "loadedModel" to make predictions -// Make predictions on test documents loadedModel.transform(test) .select("id", "text", "probability", "prediction") .collect() .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => println(s"($id, $text) --> prob=$prob, prediction=$prediction") - } + } {% endhighlight %} From ad715ba2fe2fb520d6afb7123a9dfa31f47828ee Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 10:16:49 -0800 Subject: [PATCH 09/16] small changes for @jkbradley --- docs/ml-guide.md | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 677bf7608f71..86c21c692495 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -194,13 +194,7 @@ This is useful if there are two algorithms with the `maxIter` parameter in a `Pi ## Saving and Loading Pipelines -Often times it is worth it to save a model or a pipeline to disk for later use. In Spark 1.6, a model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models such as: - -* K-Means -* Naive Bayes -* ALS -* Linear Regression -* Logistic Regression +Often times it is worth it to save a model or a pipeline to disk for later use. In Spark 1.6, a model import/export functionality was added to the Pipeline API. Most basic transformers are supported as well as some of the more basic ML models. Please refer to the algorithm's API documentation to see if saving and loading is supported. # Code examples @@ -465,13 +459,16 @@ val pipeline = new Pipeline() // Fit the pipeline to training documents. val model = pipeline.fit(training) -// now we can optionally save it to disk +// now we can optionally save the fitted pipeline to disk model.save("/tmp/spark-logistic-regression-model") +// we can also save this unfit pipeline to disk +pipeline.save("/tmp/unfit-lr-model") + // and load it back in during production -val loadedModel = Pipeline.load("/tmp/spark-logistic-regression-model") +val sameModel = Pipeline.load("/tmp/spark-logistic-regression-model") // or equivalently -val loadedModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") +val sameModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") // Prepare test documents, which are unlabeled (id, text) tuples. val test = sqlContext.createDataFrame(Seq( @@ -489,14 +486,6 @@ model.transform(test) println(s"($id, $text) --> prob=$prob, prediction=$prediction") } -// or use the "loadedModel" to make predictions -loadedModel.transform(test) - .select("id", "text", "probability", "prediction") - .collect() - .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => - println(s"($id, $text) --> prob=$prob, prediction=$prediction") - } - {% endhighlight %} From 66f692f8f3d101149b1c9bd149d164996ea8f6a7 Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 10:19:30 -0800 Subject: [PATCH 10/16] small change to revert --- docs/ml-guide.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 86c21c692495..cb4f89a65e26 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -140,7 +140,8 @@ If the `Pipeline` had more stages, it would call the `LogisticRegressionModel`'s method on the `DataFrame` before passing the `DataFrame` to the next stage. A `Pipeline` is an `Estimator`. -Thus, after a `Pipeline`'s `fit()` method runs, it produces a `PipelineModel`, which is a `Transformer`. +Thus, after a `Pipeline`'s `fit()` method runs, it produces a `PipelineModel`, which is a +`Transformer`. This `PipelineModel` is used at *test time*; the figure below illustrates this usage. From 2d0d87c63a4cfab03dcfaba1305160e4daefc6c3 Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 10:20:33 -0800 Subject: [PATCH 11/16] small revert --- docs/ml-guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index cb4f89a65e26..6131f54251d1 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -617,6 +617,7 @@ for row in selected.collect(): {% endhighlight %} + ## Example: model selection via cross-validation From da91fb22bc677632afc9d50c07c7eb9fc88c90b6 Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 10:21:07 -0800 Subject: [PATCH 12/16] small revert --- docs/ml-guide.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 6131f54251d1..9ea8e27bee05 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -954,5 +954,3 @@ model.transform(test) {% endhighlight %} - - From c596edfec097d9d8a794a6e8d0321d82c55fae89 Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 10:22:08 -0800 Subject: [PATCH 13/16] small revert again --- docs/ml-guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 9ea8e27bee05..6131f54251d1 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -954,3 +954,5 @@ model.transform(test) {% endhighlight %} + + From 9c7ad99bfe41a2a2c30cae549e6b1304c9ac0f9e Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 10:23:51 -0800 Subject: [PATCH 14/16] small changes, little by little! --- docs/ml-guide.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 6131f54251d1..a79cd8435110 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -142,7 +142,6 @@ method on the `DataFrame` before passing the `DataFrame` to the next stage. A `Pipeline` is an `Estimator`. Thus, after a `Pipeline`'s `fit()` method runs, it produces a `PipelineModel`, which is a `Transformer`. - This `PipelineModel` is used at *test time*; the figure below illustrates this usage.

@@ -559,7 +558,6 @@ Pipeline pipeline = new Pipeline() // Fit the pipeline to training documents. PipelineModel model = pipeline.fit(training); - // Prepare test documents, which are unlabeled. DataFrame test = sqlContext.createDataFrame(Arrays.asList( new Document(4L, "spark i j k"), From 06f8c2ad00b069ad05d670f90c7f1fc66c59534c Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 10:38:17 -0800 Subject: [PATCH 15/16] revert end of page --- docs/ml-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index a79cd8435110..47cf5e4cbae1 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -953,4 +953,4 @@ model.transform(test) {% endhighlight %} - + \ No newline at end of file From 10ccc22366adca8b9fde0eff200629a56ba01161 Mon Sep 17 00:00:00 2001 From: anabranch Date: Fri, 11 Dec 2015 11:55:01 -0800 Subject: [PATCH 16/16] removed 470 and 471 --- docs/ml-guide.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/ml-guide.md b/docs/ml-guide.md index 47cf5e4cbae1..44a316a07dfe 100644 --- a/docs/ml-guide.md +++ b/docs/ml-guide.md @@ -467,8 +467,6 @@ pipeline.save("/tmp/unfit-lr-model") // and load it back in during production val sameModel = Pipeline.load("/tmp/spark-logistic-regression-model") -// or equivalently -val sameModel = Pipeline.read.load("/tmp/spark-logistic-regression-model") // Prepare test documents, which are unlabeled (id, text) tuples. val test = sqlContext.createDataFrame(Seq(