SQLContext sqlContext =new SQLContext(sc);
DataFrame dataFrame = sqlContext.range(1,10);
DataFrame frame = dataFrame.select(functions.col("id"), functions.rand(10).alias("uniform"), functions.randn(25).alias("normal"));
frame.describe("uniform","normal").show();

SQLContext sqlContext =new SQLContext(sc);
DataFrame dataFrame = sqlContext.range(1,10);
DataFrame frame = dataFrame.select(functions.col("id"), functions.rand(10).alias("uniform"), functions.randn(25).alias("normal"));
frame.describe().show();

SQLContext sqlContext =new SQLContext(sc);
DataFrame dataFrame = sqlContext.range(1,10);
DataFrame frame = dataFrame.select(functions.col("id"), functions.rand(10).alias("randX"), functions.randn(10).alias("randY"));
double cov= frame.stat().cov("randX","randY");
System.out.printf("cov: %s%n", cov);

SQLContext sqlContext =new SQLContext(sc);
DataFrame dataFrame = sqlContext.range(1,10);
DataFrame frame = dataFrame.select(functions.col("id"), functions.rand(10).alias("randX"), functions.rand(10).alias("randY"));
double id_corr = frame.stat().corr("id","id");
System.out.printf("id corr: %s%n", id_corr);

SQLContext sqlContext =new SQLContext(sc);
String[] names = {
"Xiaoming",
"Xiaohong",
"Xiaogang"
};
String[] items = {
"milk",
"bread",
"butter",
"apples",
"oranges"
};

Random random =new Random();
List < Consumption > consumptions = Lists.newArrayList();
for (String name: names) {
for (int i =0; i <5; i++) {
consumptions.add(new Consumption(name, items[random.nextInt(items.length)]));
}
}
JavaRDD < Consumption > parallelize = sc.parallelize(consumptions);
DataFrame dataFrame = sqlContext.createDataFrame(parallelize, Consumption.class);
dataFrame.registerTempTable("consumption");
dataFrame.show()

dataFrame.stat().crosstab("name","item").show();

IsotonicRegressionModel model =new IsotonicRegression().setIsotonic(true).run(training);

List < Tuple2 < Object,Vector >> list = Lists.newArrayList(
new Tuple2 < >(1d, Vectors.dense(0.0d,1d)),
new Tuple2 < >(7d, Vectors.dense(2.0d,1d)),
new Tuple2 < >(10d, Vectors.dense(3.0d,1d)),
new Tuple2 < >(4d, Vectors.dense(1.0d,1d)),
new Tuple2 < >(19d, Vectors.dense(6.0d,1d))
);
JavaPairRDD < Object,Vector > rdd = jsc.parallelizePairs(list);

Gradient gradient =new LeastSquaresGradient();
Updater updater =new L1Updater();
Optimizer descent =new GradientDescent(gradient, updater);

Vector optimize = descent.optimize(rdd.rdd(), Vectors.dense(0d,0d));
System.out.println(optimize);

Tokenizer tokenizer =new Tokenizer()
.setInputCol("text")
.setOutputCol("words");
HashingTF hashingTF =new HashingTF()
.setNumFeatures(1000)
.setInputCol(tokenizer.getOutputCol())
.setOutputCol("features");
LogisticRegression lr =new LogisticRegression()
.setMaxIter(10)
.setRegParam(0.01);
Pipeline pipeline =new Pipeline()
.setStages(new PipelineStage[] {tokenizer, hashingTF, lr});

PipelineModel model = pipeline.fit(training);