Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[xdoctest] reformat example code with google style in paddle/io #55732

Merged
merged 7 commits into from
Aug 3, 2023
Merged

[xdoctest] reformat example code with google style in paddle/io #55732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c5eb488
[xdoctest] reformat example code with google style in `paddle/io`
SigureMo Jul 26, 2023
e8a6b31
preview, test=docs_preview
SigureMo Jul 26, 2023
fbce310
update example code, test=docs_preview
SigureMo Aug 1, 2023
71c80e8
update output, test=docs_preview
SigureMo Aug 2, 2023
90343f5
remove unused imports, test=docs_preview
SigureMo Aug 2, 2023
8619edd
skip some device depends apis, test=docs_preview
SigureMo Aug 2, 2023
a89232c
add skip reason, test=docs_preview
SigureMo Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 83 additions & 79 deletions python/paddle/io/dataloader/batch_sampler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,40 +58,44 @@ class BatchSampler(Sampler):

.. code-block:: python

from paddle.io import RandomSampler, BatchSampler, Dataset

# init with dataset
class RandomDataset(Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples

def __getitem__(self, idx):
image = np.random.random([784]).astype('float32')
label = np.random.randint(0, 9, (1, )).astype('int64')
return image, label

def __len__(self):
return self.num_samples

bs = BatchSampler(dataset=RandomDataset(100),
shuffle=False,
batch_size=16,
drop_last=False)

for batch_indices in bs:
print(batch_indices)

# init with sampler
sampler = RandomSampler(RandomDataset(100))
bs = BatchSampler(sampler=sampler,
batch_size=8,
drop_last=True)

for batch_indices in bs:
print(batch_indices)



>>> import numpy as np
>>> from paddle.io import RandomSampler, BatchSampler, Dataset

>>> np.random.seed(2023)
>>> # init with dataset
>>> class RandomDataset(Dataset):
... def __init__(self, num_samples):
... self.num_samples = num_samples
...
... def __getitem__(self, idx):
... image = np.random.random([784]).astype('float32')
... label = np.random.randint(0, 9, (1, )).astype('int64')
... return image, label
...
... def __len__(self):
... return self.num_samples
...
>>> bs = BatchSampler(dataset=RandomDataset(100),
... shuffle=False,
... batch_size=16,
... drop_last=False)
...
>>> for batch_indices in bs:
... print(batch_indices)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
...
[96, 97, 98, 99]
>>> # init with sampler
>>> sampler = RandomSampler(RandomDataset(100))
>>> bs = BatchSampler(sampler=sampler,
... batch_size=8,
... drop_last=True)
...
>>> for batch_indices in bs:
... print(batch_indices)
[56, 12, 68, 0, 82, 66, 91, 44]
...
[53, 17, 22, 86, 52, 3, 92, 33]
"""

def __init__(
Expand Down Expand Up @@ -203,29 +207,29 @@ class DistributedBatchSampler(BatchSampler):
Examples:
.. code-block:: python

import numpy as np

from paddle.io import Dataset, DistributedBatchSampler

# init with dataset
class RandomDataset(Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples

def __getitem__(self, idx):
image = np.random.random([784]).astype('float32')
label = np.random.randint(0, 9, (1, )).astype('int64')
return image, label

def __len__(self):
return self.num_samples

dataset = RandomDataset(100)
sampler = DistributedBatchSampler(dataset, batch_size=64)

for data in sampler:
# do something
break
>>> import numpy as np

>>> from paddle.io import Dataset, DistributedBatchSampler

>>> # init with dataset
>>> class RandomDataset(Dataset):
... def __init__(self, num_samples):
... self.num_samples = num_samples
...
... def __getitem__(self, idx):
... image = np.random.random([784]).astype('float32')
... label = np.random.randint(0, 9, (1, )).astype('int64')
... return image, label
...
... def __len__(self):
... return self.num_samples
...
>>> dataset = RandomDataset(100)
>>> sampler = DistributedBatchSampler(dataset, batch_size=64)

>>> for data in sampler:
... # do something
... break
Comment on lines +231 to +232
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

或者其他的几个 print 的地方都改成这样也不错～

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

修改了一版，整体对于 image 的都采用这个策略，因为大多数不需要 print，直接加注释 do something

其他的该加输出的也都加了

"""

def __init__(
Expand Down Expand Up @@ -339,27 +343,27 @@ def set_epoch(self, epoch):
Examples:
.. code-block:: python

import numpy as np

from paddle.io import Dataset, DistributedBatchSampler

# init with dataset
class RandomDataset(Dataset):
def __init__(self, num_samples):
self.num_samples = num_samples

def __getitem__(self, idx):
image = np.random.random([784]).astype('float32')
label = np.random.randint(0, 9, (1, )).astype('int64')
return image, label

def __len__(self):
return self.num_samples

dataset = RandomDataset(100)
sampler = DistributedBatchSampler(dataset, batch_size=64)

for epoch in range(10):
sampler.set_epoch(epoch)
>>> import numpy as np

>>> from paddle.io import Dataset, DistributedBatchSampler

>>> # init with dataset
>>> class RandomDataset(Dataset):
... def __init__(self, num_samples):
... self.num_samples = num_samples
...
... def __getitem__(self, idx):
... image = np.random.random([784]).astype('float32')
... label = np.random.randint(0, 9, (1, )).astype('int64')
... return image, label
...
... def __len__(self):
... return self.num_samples
...
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just一点小疑问，这部分如果去掉 ... 是不是也没问题

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

确实是 ～ xdoctest 可以兼容只用 >>> ，多行也不加 ... 提示。

但是，这只是 xdoctest 的特性，保不齐哪天不用 xdoctest 了，这样可能就挂了～

所以，个人建议，还是以 python 的 doctest 为最低兼容标准吧～

>>> dataset = RandomDataset(100)
>>> sampler = DistributedBatchSampler(dataset, batch_size=64)

>>> for epoch in range(10):
... sampler.set_epoch(epoch)
"""
self.epoch = epoch
Loading