Google Cloud Dataflowの切り詰められたログ

Question

Google Cloud Dataflowでコードを実行すると、「Errors」ビューにトレースバックが表示されますが、切り捨てられます。

(cee7eb663424a856): Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py", line 578, in do_work work_executor.execute() File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/executor.py", line 165, in execute op.start() File "dataflow_worker/shuffle_operations.py", line 49, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start (dataflow_worker/shuffle_operations.c:3442) def start(self): File "dataflow_worker/shuffle_operations.py", line 50, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start (dataflow_worker/shuffle_operations.c:3346) with self.scoped_start_state: File "dataflow_worker/shuffle_operations.py", line 63, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start (dataflow_worker/shuffle_operations.c:3261) with self.shuffle_source.reader() as reader: File "dataflow_worker/shuffle_operations.py", line 67, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start (dataflow_worker/shuffle_operations.c:3205) self.output(windowed_value) File "dataflow_worker/operations.py", line 139, in dataflow_worker.operations.Operation.output (dataflow_worker/operations.c:5717) cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive (dataflow_worker/operations.c:3421) cython.cast(Operation, consumer).process(windowed_value) File "dataflow_worker/shuffle_operations.py", line 220, in dataflow_worker.shuffle_operations.BatchGroupAlsoByWindowsOperation.process (dataflow_worker/shuffle_operations.c:8094) self.output(wvalue.with_value((k, wvalue.value))) File "dataflow_worker/operations.py", line 139, in dataflow_worker.operations.Operation.output (dataflow_worker/operations.c:5717) cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive (dataflow_worker/operations.c:3421) cython.cast(Operation, consumer).process(windowed_value) File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13546) with self.scoped_process_state: File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13500) self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive (apache_beam/runners/common.c:5142) self.process(windowed_value) File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7201) self.reraise_augmented(exn) File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented (apache_beam/runners/common.c:7609) raise File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7090) self._dofn_simple_invoker(element) File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker (apache_beam/runners/common.c:5288) self._process_outputs(element, self.dofn_process(element.value)) File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs (apache_beam/runners/common.c:8563) self.main_receivers.receive(windowed_value) File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive (dataflow_worker/operations.c:3421) cython.cast(Operation, consumer).process(windowed_value) File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13546) with self.scoped_process_state: File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13500) self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive (apache_beam/runners/common.c:5142) self.process(windowed_value) File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7201) self.reraise_augmented(exn) File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented (apache_beam/runners/common.c:7609) raise File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7090) self._dofn_simple_invoker(element) File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker (apache_beam/runners/common.c:5288) self._process_outputs(element, self.dofn_process(element.value)) File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs (apache_beam/runners/common.c:8563) self.main_receivers.receive(windowed_value) File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive (dataflow_worker/operations.c:3421) cython.cast(Operation, consumer).process(windowed_value) File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13546) with self.scoped_process_state: File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13500) self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive (apache_beam/runners/common.c:5142) self.process(windowed_value) File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7201) self.reraise_augmented(exn) File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented (apache_beam/runners/common.c:7609) raise File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7090) self._dofn_simple_invoker(element) File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker (apache_beam/runners/common.c:5288) self._process_outputs(element, self.dofn_process(element.value)) File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs (apache_beam/runners/common.c:8563) self.main_receivers.receive(windowed_value) File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive (dataflow_worker/operations.c:3421) cython.cast(Operation, consumer).process(windowed_value) File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13546) with self.scoped_process_state: File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13500) self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive (apache_beam/runners/common.c:5142) self.process(windowed_value) File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7201) self.reraise_augmented(exn) File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented (apache_beam/runners/common.c:7609) raise File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7090) self._dofn_simple_invoker(element) File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker (apache_beam/runners/common.c:5288) self._process_outputs(element, self.dofn_process(element.value)) File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs (apache_beam/runners/common.c:8563) self.main_receivers.receive(windowed_value) File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive (dataflow_worker/operations.c:3421) cython.cast(Operation, consumer).process(windowed_value) File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13546) with self.scoped_process_state: File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process (dataflow_worker/operations.c:13500) self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive (apache_beam/runners/common.c:5142) self.process(windowed_value) File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7201) self.reraise_augmented(exn) File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented (apache_beam/runners/common.c:7609) raise File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7090) self._dofn_simple_invoker(element) File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker (apache_beam/runners/common.c:5288) self._process_outputs(element, self.dofn_process(element.value)) File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs (apache_beam/runners/common.c:8563) self.main_receivers.receive(windowed_value) File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive (dataflow_worker/operations.c:3421) cython.cast(Operation, consumer).process(windowed_valu

それは周りの10220文字の後に...一番最後に切り捨てられ、そしてそれもまだ私の実際のコードのスタックトレースを得ていないか注意してください！

ログパネルを開いてログを見ると、私のコードを指している最後の行だけでなく、実際の例外クラスも完全に見ることができます。メッセージそのもの。

Answer 1

データフローUIに表示されるエラーは、約10k文字で切り捨てられますが、スタックドライバのログを見ると、例外全体が表示されるはずです。