Coverage for pyspark/ml/wrapper.py : 96%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
|
# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #
""" Wrapper class for a Java companion object """
def _create_from_java_class(cls, java_class, *args): """ Construct this object from given Java classname and arguments """
def _new_java_obj(java_class, *args): """ Returns a new Java object. """
def _new_java_array(pylist, java_class): """ Create a Java array of given java_class type. Useful for calling a method with a Scala Array from Python with Py4J. If the param pylist is a 2D array, then a 2D java array will be returned. The returned 2D java array is a square, non-jagged 2D array that is big enough for all elements. The empty slots in the inner Java arrays will be filled with null to make the non-jagged 2D array.
Parameters ---------- pylist : list Python list to convert to a Java Array. java_class : :py:class:`py4j.java_gateway.JavaClass` Java class to specify the type of Array. Should be in the form of sc._gateway.jvm.* (sc is a valid Spark Context).
Example primitive Java classes:
- basestring -> sc._gateway.jvm.java.lang.String - int -> sc._gateway.jvm.java.lang.Integer - float -> sc._gateway.jvm.java.lang.Double - bool -> sc._gateway.jvm.java.lang.Boolean
Returns ------- :py:class:`py4j.java_collections.JavaArray` Java Array of converted pylist. """ # If pylist is a 2D array, then a 2D java array will be created. # The 2D array is a square, non-jagged 2D array that is big enough for all elements. else:
""" Utility class to help create wrapper classes from Java/Scala implementations of pipeline components. """ #: The param values in the Java object should be #: synced with the Python wrapper in fit/transform/evaluate/copy.
""" Makes a Java param pair. """
""" Transforms the embedded params to the companion Java object. """
""" Transforms a Python ParamMap into a Java ParamMap. """
""" SPARK-10931: Temporary fix to create params that are defined in the Java obj but not here """ param = Param(self, java_param_name, java_param.doc()) setattr(param, "created_from_java_param", True) setattr(self, java_param_name, param) self._params = None # need to reset so self.params will discover new params
""" Transforms the embedded params from the companion Java object. """ # SPARK-14931: Only check set params back to avoid default params mismatch. # SPARK-10931: Temporary fix for params that have a default in Java
""" Transforms a Java ParamMap into a Python ParamMap. """
def _empty_java_param_map(): """ Returns an empty Java ParamMap reference. """
""" Transfer this instance's Params to the wrapped Java object, and return the Java object. Used for ML persistence.
Meta-algorithms such as Pipeline should override this method.
Returns ------- py4j.java_gateway.JavaObject Java object equivalent to this instance. """
def _from_java(java_stage): """ Given a Java object, create and return a Python wrapper of it. Used for ML persistence.
Meta-algorithms such as Pipeline should override this method as a classmethod. """ """ Loads Python class from its name. """ # Generate a default new instance from the stage_name class. # Load information from java_stage to the instance.
# SPARK-10931: Temporary fix so that persisted models would own params from Estimator
else: raise NotImplementedError("This Java stage cannot be loaded into Python currently: %r" % stage_name)
""" Creates a copy of this instance with the same uid and some extra params. This implementation first calls Params.copy and then make a copy of the companion Java pipeline component with extra params. So both the Python wrapper and the Java pipeline component get copied.
Parameters ---------- extra : dict, optional Extra parameters to copy to the new instance
Returns ------- :py:class:`JavaParams` Copy of this instance """
""" Clears a param from the param map if it has been explicitly set. """
""" Base class for :py:class:`Estimator`s that wrap Java/Scala implementations. """
def _create_model(self, java_model): """ Creates a model from the input Java model reference. """ raise NotImplementedError()
""" Fits a Java model to the input dataset.
Examples -------- dataset : :py:class:`pyspark.sql.DataFrame` input dataset
Returns ------- py4j.java_gateway.JavaObject fitted Java model """
""" Base class for :py:class:`Transformer`s that wrap Java/Scala implementations. Subclasses should ensure they have the transformer Java object available as _java_obj. """
""" Base class for :py:class:`Model`s that wrap Java/Scala implementations. Subclasses should inherit this class before param mix-ins, because this sets the UID from the Java model. """
""" Initialize this instance with a Java model object. Subclasses should call this constructor, initialize params, and then call _transfer_params_from_java.
This instance can be instantiated without specifying java_model, it will be assigned after that, but this scenario only used by :py:class:`JavaMLReader` to load models. This is a bit of a hack, but it is easiest since a proper fix would require MLReader (in pyspark.ml.util) to depend on these wrappers, but these wrappers depend on pyspark.ml.util (both directly and via other ML classes). """
# SPARK-10931: This is a temporary fix to allow models to own params # from estimators. Eventually, these params should be in models through # using common base classes between estimators and models.
""" (Private) Java Estimator for prediction tasks (regression and classification). """
""" (Private) Java Model for prediction tasks (regression and classification). """
def numFeatures(self): """ Returns the number of features the model was trained on. If unknown, returns -1 """
def predict(self, value): """ Predict label for the given features. """ |