Hello ,
We are trying to leverage DaskLGBMRegressor and receiving an error . The data structures are
print(x_train.info() )
<class 'dask.dataframe.core.DataFrame'>
Columns: 448 entries, CDL2CROWS_1Min to CDLXSIDEGAP3METHODS_128Min
dtypes: float64(448)None
print(y_train.info() )
<class 'dask.dataframe.core.DataFrame'>
Columns: 1 entries, close_future_512 to close_future_512
dtypes: float64(1)None
the code being executed is
model = lgb.DaskLGBMRegressor()
now = datetime.datetime.now()
dask.distributed.print("STATUS -{0} Set patams ".format(now.strftime("%Y-%m-%d %H:%M:%S")))
model.set_params(n_estimators=200)
model.set_params(max_depth=7)
model.set_params(num_leaves=16)
model.set_params(boosting_type='goss')
model.set_params(min_child_samples=25)
model.set_params(learning_rate=0.02)
model.set_params(colsample_bytree=0.9)
now = datetime.datetime.now()
dask.distributed.print("begin initial training at {}".format(now.strftime("%Y-%m-%d %H:%M:%S")))
gbm = model.fit(X=x_train,y=y_train, eval_metric= usefill_metric)
now = datetime.datetime.now()
dask.distributed.print("finished initial training at {}".format(now.strftime("%Y-%m-%d %H:%M:%S")))
print('feature importance is: ')
print(gbm.feature_importances_)
the error we get is
CancelledError Traceback (most recent call last)
Cell In[7], line 285
283 now = datetime.datetime.now()
284 dask.distributed.print("begin initial training at {}".format(now.strftime("%Y-%m-%d %H:%M:%S")))
--> 285 gbm = model.fit(X=x_train,y=y_train, eval_metric= usefill_metric)
286 now = datetime.datetime.now()
287 dask.distributed.print("finished initial training at {}".format(now.strftime("%Y-%m-%d %H:%M:%S")))
File ~/.local/lib/python3.10/site-packages/lightgbm/dask.py:1406, in DaskLGBMRegressor.fit(self, X, y, sample_weight, init_score, eval_set, eval_names, eval_sample_weight, eval_init_score, eval_metric, **kwargs)
1392 def fit( # type: ignore[override]
1393 self,
1394 X: _DaskMatrixLike,
(...)
1403 **kwargs: Any
1404 ) -> "DaskLGBMRegressor":
1405 """Docstring is inherited from the lightgbm.LGBMRegressor.fit."""
-> 1406 self._lgb_dask_fit(
1407 model_factory=LGBMRegressor,
1408 X=X,
1409 y=y,
1410 sample_weight=sample_weight,
1411 init_score=init_score,
1412 eval_set=eval_set,
1413 eval_names=eval_names,
1414 eval_sample_weight=eval_sample_weight,
1415 eval_init_score=eval_init_score,
1416 eval_metric=eval_metric,
1417 **kwargs
1418 )
1419 return self
File ~/.local/lib/python3.10/site-packages/lightgbm/dask.py:1082, in _DaskLGBMModel._lgb_dask_fit(self, model_factory, X, y, sample_weight, init_score, group, eval_set, eval_names, eval_sample_weight, eval_class_weight, eval_init_score, eval_group, eval_metric, eval_at, **kwargs)
1079 params = self.get_params(True) # type: ignore[attr-defined]
1080 params.pop("client", None)
-> 1082 model = _train(
1083 client=_get_dask_client(self.client),
1084 data=X,
1085 label=y,
1086 params=params,
1087 model_factory=model_factory,
1088 sample_weight=sample_weight,
1089 init_score=init_score,
1090 group=group,
1091 eval_set=eval_set,
1092 eval_names=eval_names,
1093 eval_sample_weight=eval_sample_weight,
1094 eval_class_weight=eval_class_weight,
1095 eval_init_score=eval_init_score,
1096 eval_group=eval_group,
1097 eval_metric=eval_metric,
1098 eval_at=eval_at,
1099 **kwargs
1100 )
1102 #self.set_params(**model.get_params()) # type: ignore[attr-defined]
1103 #self._lgb_dask_copy_extra_params(model, self) # type: ignore[attr-defined]
1105 return self
File ~/.local/lib/python3.10/site-packages/lightgbm/dask.py:694, in _train(client, data, label, params, model_factory, sample_weight, init_score, group, eval_set, eval_names, eval_sample_weight, eval_class_weight, eval_init_score, eval_group, eval_metric, eval_at, **kwargs)
692 parts = list(map(delayed, parts))
693 parts = client.compute(parts)
--> 694 wait(parts)
696 for part in parts:
697 if part.status == 'error': # type: ignore
698 # trigger error locally
File ~/.local/lib/python3.10/site-packages/distributed/client.py:5267, in wait(fs, timeout, return_when)
5265 timeout = parse_timedelta(timeout, default="s")
5266 client = default_client()
-> 5267 result = client.sync(_wait, fs, timeout=timeout, return_when=return_when)
5268 return result
File ~/.local/lib/python3.10/site-packages/distributed/utils.py:359, in SyncMethodMixin.sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
357 return future
358 else:
--> 359 return sync(
360 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
361 )
File ~/.local/lib/python3.10/site-packages/distributed/utils.py:426, in sync(loop, func, callback_timeout, *args, **kwargs)
424 if error:
425 typ, exc, tb = error
--> 426 raise exc.with_traceback(tb)
427 else:
428 return result
File ~/.local/lib/python3.10/site-packages/distributed/utils.py:399, in sync.<locals>.f()
397 future = wait_for(future, callback_timeout)
398 future = asyncio.ensure_future(future)
--> 399 result = yield future
400 except Exception:
401 error = sys.exc_info()
File ~/.local/lib/python3.10/site-packages/tornado/gen.py:767, in Runner.run(self)
765 try:
766 try:
--> 767 value = future.result()
768 except Exception as e:
769 # Save the exception for later. It's important that
770 # gen.throw() not be called inside this try/except block
771 # because that makes sys.exc_info behave unexpectedly.
772 exc: Optional[Exception] = e
File ~/.local/lib/python3.10/site-packages/distributed/client.py:5243, in _wait(fs, timeout, return_when)
5241 cancelled = [f.key for f in done if f.status == "cancelled"]
5242 if cancelled:
-> 5243 raise CancelledError(cancelled)
5245 return DoneAndNotDoneFutures(done, not_done)
CancelledError: ['dict-c99565de-7b35-489e-9356-82504a139608']
The cluster was created using EC2Cluster
