pyspark.conf

59 60 """ 61 Configuration for a Spark application. Used to set various Spark 62 parameters as key-value pairs. 63 64 Most of the time, you would create a SparkConf object with 65 C{SparkConf()}, which will load values from C{spark.*} Java system 66 properties as well. In this case, any parameters you set directly on 67 the C{SparkConf} object take priority over system properties. 68 69 For unit tests, you can also call C{SparkConf(false)} to skip 70 loading external settings and get the same configuration no matter 71 what the system properties are. 72 73 All setter methods in this class support chaining. For example, 74 you can write C{conf.setMaster("local").setAppName("My app")}. 75 76 Note that once a SparkConf object is passed to Spark, it is cloned 77 and can no longer be modified by the user. 78 """ 79

80 - def __init__(self, loadDefaults=True, _jvm=None, _jconf=None):

81 """ 82 Create a new Spark configuration. 83 84 @param loadDefaults: whether to load values from Java system 85 properties (True by default) 86 @param _jvm: internal parameter used to pass a handle to the 87 Java VM; does not need to be set by users 88 @param _jconf: Optionally pass in an existing SparkConf handle 89 to use its parameters 90 """ 91 if _jconf: 92 self._jconf = _jconf 93 else: 94 from pyspark.context import SparkContext 95 SparkContext._ensure_initialized() 96 _jvm = _jvm or SparkContext._jvm 97 self._jconf = _jvm.SparkConf(loadDefaults)

98

99 - def set(self, key, value):

100 """Set a configuration property.""" 101 self._jconf.set(key, unicode(value)) 102 return self

103

104 - def setIfMissing(self, key, value):

105 """Set a configuration property, if not already set.""" 106 if self.get(key) is None: 107 self.set(key, value) 108 return self

109

110 - def setMaster(self, value):

111 """Set master URL to connect to.""" 112 self._jconf.setMaster(value) 113 return self

114

115 - def setAppName(self, value):

116 """Set application name.""" 117 self._jconf.setAppName(value) 118 return self

119

120 - def setSparkHome(self, value):

121 """Set path where Spark is installed on worker nodes.""" 122 self._jconf.setSparkHome(value) 123 return self

124

125 - def setExecutorEnv(self, key=None, value=None, pairs=None):

126 """Set an environment variable to be passed to executors.""" 127 if (key is not None and pairs is not None) or (key is None and pairs is None): 128 raise Exception("Either pass one key-value pair or a list of pairs") 129 elif key is not None: 130 self._jconf.setExecutorEnv(key, value) 131 elif pairs is not None: 132 for (k, v) in pairs: 133 self._jconf.setExecutorEnv(k, v) 134 return self

135

136 - def setAll(self, pairs):

137 """ 138 Set multiple parameters, passed as a list of key-value pairs. 139 140 @param pairs: list of key-value pairs to set 141 """ 142 for (k, v) in pairs: 143 self._jconf.set(k, v) 144 return self

145

146 - def get(self, key, defaultValue=None):

147 """Get the configured value for some key, or return a default otherwise.""" 148 if defaultValue is None: # Py4J doesn't call the right get() if we pass None 149 if not self._jconf.contains(key): 150 return None 151 return self._jconf.get(key) 152 else: 153 return self._jconf.get(key, defaultValue)

154

155 - def getAll(self):

156 """Get all values as a list of key-value pairs.""" 157 pairs = [] 158 for elem in self._jconf.getAll(): 159 pairs.append((elem._1(), elem._2())) 160 return pairs

161

162 - def contains(self, key):

163 """Does this configuration contain a given key?""" 164 return self._jconf.contains(key)

165

166 - def toDebugString(self):

167 """ 168 Returns a printable version of the configuration, as a list of 169 key=value pairs, one per line. 170 """ 171 return self._jconf.toDebugString()

Source Code for Module pyspark.conf