diff --git a/.buildinfo b/.buildinfo new file mode 100644 index 0000000..feab728 --- /dev/null +++ b/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. +config: 9b73e76feb3affc408e7e6f8f7f34104 +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/.doctrees/bianca.doctree b/.doctrees/bianca.doctree new file mode 100644 index 0000000..32a3c9b Binary files /dev/null and b/.doctrees/bianca.doctree differ diff --git a/.doctrees/clusters.doctree b/.doctrees/clusters.doctree new file mode 100644 index 0000000..62ad1f1 Binary files /dev/null and b/.doctrees/clusters.doctree differ diff --git a/.doctrees/courses.doctree b/.doctrees/courses.doctree new file mode 100644 index 0000000..bbf316c Binary files /dev/null and b/.doctrees/courses.doctree differ diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle new file mode 100644 index 0000000..c82918a Binary files /dev/null and b/.doctrees/environment.pickle differ diff --git a/.doctrees/exercises.doctree b/.doctrees/exercises.doctree new file mode 100644 index 0000000..167a466 Binary files /dev/null and b/.doctrees/exercises.doctree differ diff --git a/.doctrees/hpc.doctree b/.doctrees/hpc.doctree new file mode 100644 index 0000000..4753e91 Binary files /dev/null and b/.doctrees/hpc.doctree differ diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree new file mode 100644 index 0000000..00b0c64 Binary files /dev/null and b/.doctrees/index.doctree differ diff --git a/.doctrees/intro.doctree b/.doctrees/intro.doctree new file mode 100644 index 0000000..e99f23f Binary files /dev/null and b/.doctrees/intro.doctree differ diff --git a/.doctrees/intro_online.doctree b/.doctrees/intro_online.doctree new file mode 100644 index 0000000..49201fc Binary files /dev/null and b/.doctrees/intro_online.doctree differ diff --git a/.doctrees/linux.doctree b/.doctrees/linux.doctree new file mode 100644 index 0000000..980a1e5 Binary files /dev/null and b/.doctrees/linux.doctree differ diff --git a/.doctrees/linux_basics.doctree b/.doctrees/linux_basics.doctree new file mode 100644 index 0000000..06a903c Binary files /dev/null and b/.doctrees/linux_basics.doctree differ diff --git a/.doctrees/linux_cases.doctree b/.doctrees/linux_cases.doctree new file mode 100644 index 0000000..1328bbc Binary files /dev/null and b/.doctrees/linux_cases.doctree differ diff --git a/.doctrees/login.doctree b/.doctrees/login.doctree new file mode 100644 index 0000000..714ad2a Binary files /dev/null and b/.doctrees/login.doctree differ diff --git a/.doctrees/login2.doctree b/.doctrees/login2.doctree new file mode 100644 index 0000000..fcc0373 Binary files /dev/null and b/.doctrees/login2.doctree differ diff --git a/.doctrees/naiss.doctree b/.doctrees/naiss.doctree new file mode 100644 index 0000000..4897c3f Binary files /dev/null and b/.doctrees/naiss.doctree differ diff --git a/.doctrees/overview.doctree b/.doctrees/overview.doctree new file mode 100644 index 0000000..ff33511 Binary files /dev/null and b/.doctrees/overview.doctree differ diff --git a/.doctrees/overview2.doctree b/.doctrees/overview2.doctree new file mode 100644 index 0000000..a1b303c Binary files /dev/null and b/.doctrees/overview2.doctree differ diff --git a/.doctrees/rackham.doctree b/.doctrees/rackham.doctree new file mode 100644 index 0000000..8feb02d Binary files /dev/null and b/.doctrees/rackham.doctree differ diff --git a/.doctrees/slurm_intro.doctree b/.doctrees/slurm_intro.doctree new file mode 100644 index 0000000..727e75c Binary files /dev/null and b/.doctrees/slurm_intro.doctree differ diff --git a/.doctrees/snic.doctree b/.doctrees/snic.doctree new file mode 100644 index 0000000..895ad87 Binary files /dev/null and b/.doctrees/snic.doctree differ diff --git a/.doctrees/software.doctree b/.doctrees/software.doctree new file mode 100644 index 0000000..7a9f5b6 Binary files /dev/null and b/.doctrees/software.doctree differ diff --git a/.doctrees/specific.doctree b/.doctrees/specific.doctree new file mode 100644 index 0000000..27e5ba5 Binary files /dev/null and b/.doctrees/specific.doctree differ diff --git a/.doctrees/start.doctree b/.doctrees/start.doctree new file mode 100644 index 0000000..d99250a Binary files /dev/null and b/.doctrees/start.doctree differ diff --git a/.doctrees/summary.doctree b/.doctrees/summary.doctree new file mode 100644 index 0000000..632eca5 Binary files /dev/null and b/.doctrees/summary.doctree differ diff --git a/.doctrees/userguide.doctree b/.doctrees/userguide.doctree new file mode 100644 index 0000000..38b5ce7 Binary files /dev/null and b/.doctrees/userguide.doctree differ diff --git a/.doctrees/working.doctree b/.doctrees/working.doctree new file mode 100644 index 0000000..c7432e7 Binary files /dev/null and b/.doctrees/working.doctree differ diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/_images/Bild1.png b/_images/Bild1.png new file mode 100644 index 0000000..de5c886 Binary files /dev/null and b/_images/Bild1.png differ diff --git a/_images/IMG_5111.jpeg b/_images/IMG_5111.jpeg new file mode 100644 index 0000000..cefdc4e Binary files /dev/null and b/_images/IMG_5111.jpeg differ diff --git a/_images/Mac_terminal.png b/_images/Mac_terminal.png new file mode 100644 index 0000000..11f80b0 Binary files /dev/null and b/_images/Mac_terminal.png differ diff --git a/_images/Thinlinc2.jpg b/_images/Thinlinc2.jpg new file mode 100644 index 0000000..71fef53 Binary files /dev/null and b/_images/Thinlinc2.jpg differ diff --git a/_images/biancaorganisation-01.png b/_images/biancaorganisation-01.png new file mode 100644 index 0000000..5309a2c Binary files /dev/null and b/_images/biancaorganisation-01.png differ diff --git a/_images/cat.png b/_images/cat.png new file mode 100644 index 0000000..49f97b3 Binary files /dev/null and b/_images/cat.png differ diff --git a/_images/caution.png b/_images/caution.png new file mode 100644 index 0000000..e872b27 Binary files /dev/null and b/_images/caution.png differ diff --git a/_images/cross.png b/_images/cross.png new file mode 100644 index 0000000..5e95f76 Binary files /dev/null and b/_images/cross.png differ diff --git a/_images/edit.png b/_images/edit.png new file mode 100644 index 0000000..73b7509 Binary files /dev/null and b/_images/edit.png differ diff --git a/_images/flavours.png b/_images/flavours.png new file mode 100644 index 0000000..43689a7 Binary files /dev/null and b/_images/flavours.png differ diff --git a/_images/folders.png b/_images/folders.png new file mode 100644 index 0000000..2dae625 Binary files /dev/null and b/_images/folders.png differ diff --git a/_images/fz3_osx_main.png b/_images/fz3_osx_main.png new file mode 100644 index 0000000..bf32dfb Binary files /dev/null and b/_images/fz3_osx_main.png differ diff --git a/_images/gnu.png b/_images/gnu.png new file mode 100644 index 0000000..449d858 Binary files /dev/null and b/_images/gnu.png differ diff --git a/_images/head.png b/_images/head.png new file mode 100644 index 0000000..54651d4 Binary files /dev/null and b/_images/head.png differ diff --git a/_images/images.jfif b/_images/images.jfif new file mode 100644 index 0000000..ecba398 Binary files /dev/null and b/_images/images.jfif differ diff --git a/_images/mermaid-046a34d4a7c3df57cf3899fe2f612f27a718b9e9.png b/_images/mermaid-046a34d4a7c3df57cf3899fe2f612f27a718b9e9.png new file mode 100644 index 0000000..e77d54f Binary files /dev/null and b/_images/mermaid-046a34d4a7c3df57cf3899fe2f612f27a718b9e9.png differ diff --git a/_images/mermaid-34a9f1fb84f529a9f83dc50768b11e05430057fa.png b/_images/mermaid-34a9f1fb84f529a9f83dc50768b11e05430057fa.png new file mode 100644 index 0000000..0861328 Binary files /dev/null and b/_images/mermaid-34a9f1fb84f529a9f83dc50768b11e05430057fa.png differ diff --git a/_images/mermaid-b59416dfac2620d7679ac3fa5aece5f05e93534a.png b/_images/mermaid-b59416dfac2620d7679ac3fa5aece5f05e93534a.png new file mode 100644 index 0000000..615eec7 Binary files /dev/null and b/_images/mermaid-b59416dfac2620d7679ac3fa5aece5f05e93534a.png differ diff --git a/_images/mermaid-cd1265279c453823b4cb3aeee063da2eec670533.png b/_images/mermaid-cd1265279c453823b4cb3aeee063da2eec670533.png new file mode 100644 index 0000000..a001c80 Binary files /dev/null and b/_images/mermaid-cd1265279c453823b4cb3aeee063da2eec670533.png differ diff --git a/_images/mobax.jpg b/_images/mobax.jpg new file mode 100644 index 0000000..80101aa Binary files /dev/null and b/_images/mobax.jpg differ diff --git a/_images/mobax_start.jpg b/_images/mobax_start.jpg new file mode 100644 index 0000000..a92fbb5 Binary files /dev/null and b/_images/mobax_start.jpg differ diff --git a/_images/mobax_start1.jpg b/_images/mobax_start1.jpg new file mode 100644 index 0000000..7caf635 Binary files /dev/null and b/_images/mobax_start1.jpg differ diff --git a/_images/mv_inbox.png b/_images/mv_inbox.png new file mode 100644 index 0000000..3ff6b3d Binary files /dev/null and b/_images/mv_inbox.png differ diff --git a/_images/node.png b/_images/node.png new file mode 100644 index 0000000..235b4ed Binary files /dev/null and b/_images/node.png differ diff --git a/_images/nodes.png b/_images/nodes.png new file mode 100644 index 0000000..073dd2a Binary files /dev/null and b/_images/nodes.png differ diff --git a/_images/permission.png b/_images/permission.png new file mode 100644 index 0000000..bae5ee5 Binary files /dev/null and b/_images/permission.png differ diff --git a/_images/pingvin.png b/_images/pingvin.png new file mode 100644 index 0000000..b2900c2 Binary files /dev/null and b/_images/pingvin.png differ diff --git a/_images/plantuml-7e4c7f5a44bfa5548ada11af24e62e3e3d70a989.png b/_images/plantuml-7e4c7f5a44bfa5548ada11af24e62e3e3d70a989.png new file mode 100644 index 0000000..0555b42 Binary files /dev/null and b/_images/plantuml-7e4c7f5a44bfa5548ada11af24e62e3e3d70a989.png differ diff --git a/_images/program_flags.png b/_images/program_flags.png new file mode 100644 index 0000000..573d635 Binary files /dev/null and b/_images/program_flags.png differ diff --git a/_images/putty.jpg b/_images/putty.jpg new file mode 100644 index 0000000..55e1172 Binary files /dev/null and b/_images/putty.jpg differ diff --git a/_images/queue1.png b/_images/queue1.png new file mode 100644 index 0000000..259e4bb Binary files /dev/null and b/_images/queue1.png differ diff --git a/_images/queue2.png b/_images/queue2.png new file mode 100644 index 0000000..5837244 Binary files /dev/null and b/_images/queue2.png differ diff --git a/_images/queue3.png b/_images/queue3.png new file mode 100644 index 0000000..055b64c Binary files /dev/null and b/_images/queue3.png differ diff --git a/_images/screen.png b/_images/screen.png new file mode 100644 index 0000000..1a93074 Binary files /dev/null and b/_images/screen.png differ diff --git a/_images/shell.jpg b/_images/shell.jpg new file mode 100644 index 0000000..bf38e50 Binary files /dev/null and b/_images/shell.jpg differ diff --git a/_images/tab.png b/_images/tab.png new file mode 100644 index 0000000..60c59ec Binary files /dev/null and b/_images/tab.png differ diff --git a/_images/tail.png b/_images/tail.png new file mode 100644 index 0000000..792796d Binary files /dev/null and b/_images/tail.png differ diff --git a/_images/terminal.png b/_images/terminal.png new file mode 100644 index 0000000..5306fd8 Binary files /dev/null and b/_images/terminal.png differ diff --git a/_images/unix_architecture.jpg b/_images/unix_architecture.jpg new file mode 100644 index 0000000..6c94d6d Binary files /dev/null and b/_images/unix_architecture.jpg differ diff --git a/_images/uppmax-light2.jpg b/_images/uppmax-light2.jpg new file mode 100644 index 0000000..88bfc47 Binary files /dev/null and b/_images/uppmax-light2.jpg differ diff --git a/_images/wildcards_bear.png b/_images/wildcards_bear.png new file mode 100644 index 0000000..6d0ed12 Binary files /dev/null and b/_images/wildcards_bear.png differ diff --git a/_images/xeyes.png b/_images/xeyes.png new file mode 100644 index 0000000..80126f6 Binary files /dev/null and b/_images/xeyes.png differ diff --git a/_plantuml/7e/7e4c7f5a44bfa5548ada11af24e62e3e3d70a989.png b/_plantuml/7e/7e4c7f5a44bfa5548ada11af24e62e3e3d70a989.png new file mode 100644 index 0000000..0555b42 Binary files /dev/null and b/_plantuml/7e/7e4c7f5a44bfa5548ada11af24e62e3e3d70a989.png differ diff --git a/_sources/bianca.md.txt b/_sources/bianca.md.txt new file mode 100644 index 0000000..d1f0d72 --- /dev/null +++ b/_sources/bianca.md.txt @@ -0,0 +1,102 @@ +# NAISS SENS and Bianca +```{objectives} +- We'll briefly get an overview of kinds of sensitive data +- ... and the **Bianca** system +``` + +```{Admonition} The Bianca workshop +- [Bianca workshop material](https://uppmax.github.io/bianca_workshop/) +- Next time given later this spring +``` + +```{instructor-note} +- Approx timing: 14.45-15.30 (15 min Coffee break) + - Can perhaps be squeezed to 15 minutes after the coffee break +- Information +``` + +```{seealso} +- Bianca portal: http://docs.uppmax.uu.se/cluster_guides/bianca_portal/ +``` + +## Sensitive personal data + +- +- Traced to now living persons, e.g. + - human genomic data + - images/videos containing persons + - health registry (health data records from healthcare providers) +- More about sensitive data + - [GDPR](https://gdpr.eu/) + - [Data protection](https://ec.europa.eu/info/law/law-topic/data-protection_en) + - [Skydd av personuppgifter](https://ec.europa.eu/info/law/law-topic/data-protection_sv) + +- When in doubt, contact your university's [data protection officer](https://www.uppmax.uu.se/support/faq/general-miscellaneous-faq/sensitive+data+questions/). +- Generally, there must be a [Data Processing Agreement](https://www.uppmax.uu.se/support/faq/general-miscellaneous-faq/how-to-establish-a-puba-with-uu/) between UU and the data controlling university. + +## Apply for project +[Open NAISS SENS Rounds](https://supr.naiss.se/round/open_type/?type=NAISS+SENS) + +## Bianca +- Bianca is a great platform for computationally intensive research on sensitive personal data. It can also be useful for: + - national and international collaboration on sensitive personal data (without a high compute need) + - other types of sensitive data +- Bianca is not good for: + - storing data + - publishing data + - unless the dataset is very popular among Bianca users, e.g. [Swegen](https://snd.gu.se/en/catalogue/study/ext0285), [SIMPLER](https://www.simpler4health.se/) + + +## Bianca's design + +- Bianca was designed to: + - make accidental data leaks difficult + - make correct data management as easy as possible + - emulate the HPC cluster environment that SNIC/NAISS users were familiar with + - provide a maximum amount of resources + - satisfy regulations. + +### Bianca has no Internet +... but we have “solutions” + +![Image](./img/biancaorganisation-01.png) + +- Bianca is only accessible from within Sunet (i.e. from university networks). +- Use VPN outside Sunet. [Link to VPN for UU](https://mp.uu.se/en/web/info/stod/it-telefoni/it-support/network-on-campus/vpn-service) + - You can get VPN credentials from all Swedish universities. + +
+ +- The whole Bianca cluster (blue) contains hundreds of virtual project clusters (green), each of which is isolated from each other and the Internet. +- Data can be transferred to or from a virtual project cluster through the Wharf, which is a special file area that is visible from the Internet. + + +```{seealso} +- [Bianca portal in documentation](http://docs.uppmax.uu.se/cluster_guides/bianca_portal/) +- ThinLinc +- Data transfers +- Software +``` + +## Introduction course +- [Overview](https://uppmax.github.io/bianca_workshop/overview/) +- [Log in](https://uppmax.github.io/bianca_workshop/login_bianca/) +- [Command-line intro](https://uppmax.github.io/bianca_workshop/commandline/) +- [Module system](https://uppmax.github.io/bianca_workshop/modules1/) +- [Transfering files](https://uppmax.github.io/bianca_workshop/transfer) +- [More on NAISS-SENS](https://uppmax.github.io/bianca_workshop/naiss-sens-bianca/) +- [The compute nodes](https://uppmax.github.io/bianca_workshop/slurm-intro/) +- [Software and package installation](https://uppmax.github.io/bianca_workshop/install/) + + + + + ```{keypoints} + - If you handle sensitive data, apply for a NAISS-SENS project + - SENS projects will get accounts on Bianca + - Bianca has no internet itself but there are solutions like: + - wharf + - transit server + - many installed software + - Ask support if you need additional software tools + ``` diff --git a/_sources/clusters.md.txt b/_sources/clusters.md.txt new file mode 100644 index 0000000..cfbfd4a --- /dev/null +++ b/_sources/clusters.md.txt @@ -0,0 +1,34 @@ + + +# Clusters + + +We have a number of compute clusters: + + Rackham, reserved for SNIC projects + Snowy, reserved for SNIC projects + Miarka, reserved for Scilifelab production + Bianca, a part of SNIC-SENS + UPPMAX Cloud, a part of SNIC Science Cloud + +The storage systems we have provide a total volume of about 20 PB, the equivalent of nearly 15 billion 3.5-inch floppy disks or 40,000 years of 128-bit encoded music. Read more on the storage systems page. + + +## Rackham +[Rackham](https://www.uppmax.uu.se/resources/systems/the-rackham-cluster/) + +## Snowy +[Snowy](https://www.uppmax.uu.se/resources/systems/the-snowy-cluster/) + +## Bianca +[Bianca](https://www.uppmax.uu.se/resources/systems/the-bianca-cluster/) + +## Miarka +[Miarka](https://www.uppmax.uu.se/resources/systems/miarka-cluster/) + +## The UPPMAX Cloud +[UPPMAX cloud](https://www.uppmax.uu.se/resources/systems/the-uppmax-cloud/) + +## System usage +[System usage](https://www.uppmax.uu.se/resources/system-usage/) + diff --git a/_sources/courses.md.txt b/_sources/courses.md.txt new file mode 100644 index 0000000..b319c4d --- /dev/null +++ b/_sources/courses.md.txt @@ -0,0 +1,70 @@ +# Courses and workshops + + +```{instructor-note} +- Approx timing: 14.40-14:50 +- Theory +``` + +## Introductory Linux courses +### Intro to UPPMAX (This one) +- [Intro to UPPMAX](https://www.uppmax.uu.se/support/courses-and-workshops/introductory-course) + +### Awk workshop +- [Intro to Awk](https://www.uppmax.uu.se/support/courses-and-workshops/awk-workshop/) +- Jan 22-23 + +## Other UPPMAX courses +### Intro to Bianca +- [Introduction to Bianca](https://www.uppmax.uu.se/support/courses-and-workshops/bianca-workshop-2023/) +- [Course material](https://uppmax.github.io/bianca_workshop/) +- Once in spring and once in fall + +### Basic Singularity +- [Basic Singularity](https://www.uppmax.uu.se/support/courses-and-workshops/singularity-workshop-announcement/) +- Feb 13 + +### Introduction to running R, Python, and Julia in HPC +- [Introduction to running R, Python, and Julia in HPC](https://www.uppmax.uu.se/support/courses-and-workshops/r-python-julia) +- **Focus on packages and using Slurm for the jobs** +- March 12-14 + +### Git course +- [Last year's material](https://www.hpc2n.umu.se/node/982) +- Nov 13-17 + +### Python at UPPMAX and HPC2N +- [Using Python in an HPC environment](https://uppmax.github.io/HPC-python/index.html) +- Also containing parallelism, Machine learning and GPU +- May/June (Date not set) + +### Programming formalisms +- [Programming formalisms](https://www.uppmax.uu.se/support/courses-and-workshops/programming-formalisms/) +- A bit more focus on parallelism and GPUs and ML +- May/June (date not set) + +## Courses at other NAISS/SNIC Centers +- [SNIC training](http://snicdocs.nsc.liu.se/wiki/Training) + +## CodeRefinery +- [Tools workshop](https://coderefinery.github.io/2023-09-19-workshop/) +- March 12-14, 19-22? + +## ENCCS (EuroCC National Competence Centre Sweden +- [Lessons](https://enccs.se/lessons/) +- [Upcoming events](https://enccs.se/events) + +## Links to extra material +### Data Management Seminar +- [Data Management Seminar](https://www.uppmax.uu.se/support/courses-and-workshops/#tocjump_046728919125111124_0) + +### Python +- [Python at Codecademy](https://www.codecademy.com/) +- [Python at Learn X in Y minutes](https://learnxinyminutes.com/docs/python/) +- [Python village](https://rosalind.info/problems/list-view/?location=python-village) +### General Bioinformatics +- [General Bioinformatics](https://rosalind.info/problems/list-view/) +### Swedish Science Cloud +- [Swedish Science Cloud](https://github.com/SNICScienceCloud/technical-training) + + diff --git a/_sources/exercises.rst.txt b/_sources/exercises.rst.txt new file mode 100644 index 0000000..e73b998 --- /dev/null +++ b/_sources/exercises.rst.txt @@ -0,0 +1,6 @@ +Exercises +========= + +.. exerciselist}:: + + :include: exercise challenge solution instructor-note diff --git a/_sources/hpc.md.txt b/_sources/hpc.md.txt new file mode 100644 index 0000000..c58f46f --- /dev/null +++ b/_sources/hpc.md.txt @@ -0,0 +1,127 @@ +# High Performance Computing — HPC + +```{objectives} +- Let's recap and go a little further into the UPPMAX hardware! +``` + +```{seealso} +[Uppmax clusters](http://docs.uppmax.uu.se/cluster_guides/uppmax_cluster/) +``` + +```{instructor-note} +- Approx timing: 11.35-11:40 +- Information +``` + +## HPC, HTC and MTC + +- The Buzz word is **HPC or High Performance Computing**, but this is rather narrow focusing on fast calculation, i.e. processors and parallelism +- Many of your projects are more focusing on high throughput, large memory demands and many tasks. +- Here is a list of the three most common **Computing paradigms**: + +- **HPC**: High Performance Computing — Focus on floating point operations per second (**FLOPS**, flops or flop/s) + - characterized as needing large amounts of computing power for short periods of time +- **HTC**: High-Throughput Computing — + - operations or **jobs per month or per year**. + - more interested in how many jobs can be completed over a long period of time instead of how fast. + - independent, sequential jobs that can be individually scheduled o +- **MTC**: Many-task Computing — emphasis of using many computing resources over short periods of time to accomplish many computational tasks + - bridge the gap between HTC and HPC. + - reminiscent of HTC, but including both dependent and independent tasks), where the primary metrics are measured in seconds (e.g. **FLOPS**, tasks/s, **MB/s** **I/O rates**), as opposed to operations (e.g. jobs) per month. + - high-performance computations comprising multiple distinct activities, coupled via file system operations. + + + +## What is a cluster? +- A network of computers, each computer working as a node. + +- From small scale RaspberryPi cluster... + +![RaspBerry](./img/IMG_5111.jpeg) + +- To supercomputers like Rackham. + +![Rackham](./img/uppmax-light2.jpg) + +- Each node contains several processor cores and RAM and a local disk called scratch. + +![Node](./img/node.png) + +- The user logs in to login nodes via Internet through ssh or Thinlinc. + + - Here the file management and lighter data analysis can be performed. + +![RaspBerry](./img/nodes.png) + +![RaspBerry](./img/Bild1.png) + +- The calculation nodes has to be used for intense computing. + - "Normal" softwares use one core. + - Parallelized software can utilize several cores or even several nodes. Keywords signalizing this are e.g.: + - "multi-threaded", "MPI", "distributed memory", "openMP", "shared memory". + - To let your software run on the calculation nodes + - start an "interactive session" or + - "submit a batch job". + - More about this in today's introduction to jobs. + +## Storage basics +- All nodes can access: + - Your home directory on Domus or Castor + - Your project directories on Crex or Castor + - Its own local scratch disk (2-3 TB) + +- If you’re reading/writing a file once, use a directory on Crex or Castor +- If you’re reading/writing a file many times... + - Copy the file to ”scratch”, the node local disk: + ``` + cp myFile $SNIC_TMP + ``` + + +## The UPPMAX hardware + +### Clusters + +- We have a number of compute clusters: + + - [Rackham](http://docs.uppmax.uu.se/cluster_guides/rackham_portal/) +, reserved for SNIC projects + - [Snowy](http://docs.uppmax.uu.se/cluster_guides/snowy_portal/), GPU, long jobs reserved for UPPMAX projects and Education + - [Bianca](http://docs.uppmax.uu.se/cluster_guides/bianca_portal/) +, a part of SNIC-SENS + - [Miarka](https://www.uppmax.uu.se/resources/systems/miarka-cluster/), reserved for Scilifelab production + - [UPPMAX cloud](https://www.uppmax.uu.se/resources/systems/the-uppmax-cloud/), a part of SNIC Science Cloud + +- [User guides](http://docs.uppmax.uu.se/) + +- The storage systems we have provide a total volume of about 25 PB, the equivalent of 50,000 years of 128-bit encoded music. Read more on the [storage systems page](https://www.uppmax.uu.se/resources/systems/storage-systems/). + +### UPPMAX storage system names (projects & home directories) +- Rackham storage: Crex & Domus +- Bianca storage: Castor & Cygnus +- NGI production system (Miarka): Vulpes +- NGI delivery server: Grus +- Off-load storage: Lutra + +### System usage +[System usage](https://www.uppmax.uu.se/resources/system-usage/) + +- More about the systems can be found at the [System resources page](https://www.uppmax.uu.se/resources/systems/) + + +### A little bit more about Snowy + +- [User guide](http://docs.uppmax.uu.se/cluster_guides/snowy_portal/). + - There is a [local compute round](https://supr.naiss.se/round/uppmaxcompute2023/) for UU users applying for Snowy in SUPR. + - GU (courses) applications (including GU GPU usage) are not done in SUPR, but are supposed to be routed through the service desk. + - The details can be found at the [Getting started page](https://www.uppmax.uu.se/support/getting-started/course-projects/). + +### About Bianca? +- Wait for it! + + +```{keypoints} +- UPPMAX has several clusters + - each having its focus and limitation or possibilites + - access is determined by type of project +``` diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt new file mode 100644 index 0000000..0b8c271 --- /dev/null +++ b/_sources/index.rst.txt @@ -0,0 +1,76 @@ +.. uppmax_docs documentation master file, created by + sphinx-quickstart on Tue Apr 26 19:46:46 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Introduction to UPPMAX — first day lessons +========================================== + +Here you will find *some* of the documentation for the UPPMAX resources in an easy to find place. + +We will present and refer a lot to our new documentation at http://docs.uppmax.uu.se/ + +.. toctree:: + :maxdepth: 2 + :caption: Intro + + intro_online.md + +.. toctree:: + :maxdepth: 2 + :caption: Overview + + overview.md + +.. toctree:: + :maxdepth: 2 + :caption: Get started + + login2.md + linux_cases.md + +.. toctree:: + :maxdepth: 2 + :caption: More about UPPMAX + + overview2.md + hpc.md + naiss.md + +.. toctree:: + :maxdepth: 2 + :caption: Using UPPMAX + + software.md + slurm_intro.md + working.md + +.. toctree:: + :maxdepth: 2 + :caption: NAISS SENS + + bianca.md + +.. toctree:: + :maxdepth: 2 + :caption: Courses and workshops + + courses.md + +.. toctree:: + :maxdepth: 2 + :caption: Summary + + summary.md + + + + + +.. + Indices and tables + ================== + + * :ref:`genindex` + * :ref:`modindex` + * :ref:`search` diff --git a/_sources/intro.md.txt b/_sources/intro.md.txt new file mode 100644 index 0000000..b7f2bf1 --- /dev/null +++ b/_sources/intro.md.txt @@ -0,0 +1,108 @@ +# Welcome on-site! + +```{admonition} Course punchline +- Move from being a Linux novice to an UPPMAX expert. +``` + +```{admonition} Learning outcomes + - Get overview of UPPMAX and NAISS + - Work with your files in a LINUX environment + - Learn to use the UPPMAX resources in a good way + - Learn the basics in the Python programming language +``` + +## Lesson material +- First day will be on this [github website](https://uppmax.github.io/uppmax_intro/overview.html) +- All other days can be reached at the [main course page](https://www.uppmax.uu.se/support/courses-and-workshops/introductory-course-summer-2023/) + +## Questions + +- Questions very much related to the present talk during the lessons + - best posed directly by voice + +- Questions requiring more effort to explain or answer + - Use the HackMD session: + +## Type along +- You will be able to follow along using the built-in terminals in all OS:s. + - Linux/Mac Terminal + - Windows: Powershell or command prompt +- For your day-to-day work you may benefit from other applications, see the [Login session](https://uppmax.github.io/uppmax_intro/login2.html) + +## Exercises +- Exercises may be solved individually or in small groups in classroom + +## Example of arrangement on your screen +- HackMD +- lesson material +- your own terminal + + +## Lesson Modules +### Linux I at 9-12 Monday +Links in the left menu +- Overview +- Logging in to UPPMAX +- Linux basics +- Using the command line + +### Intro to UPPMAX 13-16 Monday +Links in the left menu +- Organisation +- HPC computing +- NAISS +- Using software and tools +- Submitting jobs to calculation nodes (intro) +- Working on UPPMAX +- Courses and workshops +- Short intro to **Bianca and NAISS-SENS** for the interested + + +### Linux II 9-12 Tuesday +[Linux II pdf](https://www.uppmax.uu.se/digitalAssets/560/c_560271-l_1-k_uppmax-linux-ii.pptx.pdf) +- Using the command line level 2 +- Input, output and pipes +- Wildcards +- The find command +- Links +### Linux III 13-16 Tuesday +[Linux III pdf](https://www.uppmax.uu.se/digitalAssets/560/c_560271-l_1-k_uppmax-linux-iii.pptx.pdf) +- The grep command +- Using the command line level 3 +- Variables, parameters and $(...) +- Filenames +### Bash Scripts 9-12 Wednesday +[BASH scripts pdf](https://www.uppmax.uu.se/digitalAssets/560/c_560271-l_1-k_uppmax-bash-scripts.pptx.pdf) +- Bash scripting, why's and basics +- Parameters +- The IF statement +- File tests +- FOR and WHILE loops +- Background processes and job control +### Slurm at UPPMAX 13-16 Wednesday +[Slurm pdf](https://www.uppmax.uu.se/digitalAssets/560/c_560271-l_1-k_uppmax-slurm-2022.pdf) +- The workload manager +- Quick testing and debugging as interactive jobs +- Monitoring and modifying jobs +- Priority +- Efficiency +- Script and workflow examples +- Parallel computing and GPU +- Snakemake and nextflow +### Python I 9-12 Thursday +[Python material](https://uppmax.github.io/uppmax_intro_python/) +- Introduction +- Getting started +- Data types +- Control flow statements +### Python II 13-16 Thursday +[Python material](https://uppmax.github.io/uppmax_intro_python/) +- Functions +- Modules +- Command line arguments and IO +- Object-oriented Python (bonus) + + +## Icebreaker in HackMD +- + diff --git a/_sources/intro_online.md.txt b/_sources/intro_online.md.txt new file mode 100644 index 0000000..c98865b --- /dev/null +++ b/_sources/intro_online.md.txt @@ -0,0 +1,145 @@ +# Welcome on-line! + +```{admonition} Course punchline +- Move from being a Linux novice to an UPPMAX expert. +``` + +```{admonition} Course aim +- UPPMAX application experts want to share their skills in a 4-day series of lectures. +- If you already have the fundamentals down, you are still sure to enjoy the tips and tricks in the later parts of the course. +- Hence we've made it possible to sign up only for the days that are interesting to you. +``` + +```{admonition} Learning outcomes + - Get overview of UPPMAX and NAISS + - Work with your files in a LINUX environment + - Learn to use the UPPMAX resources in a good way + - Learn the basics in the Python programming language +``` + +## Lesson material +- First day will be on this [github website](intro.md) +- All other days each topic can be reached from the [course main page](https://www.uppmax.uu.se/support/courses-and-workshops/uppmax-introductory-course) + +## New documentation page +- We will refer a lot to our [new documentation website](http://docs.uppmax.uu.se/) + +## Questions + +### Questions very much related to the present talk during the lessons +- Online chat or raised hand + +### Questions requiring more effort to explain or answer +- Use the [HackMD session](https://hackmd.io/@UPPMAX/Intro-Course-QaA) + + +## Exercises +- Exercises may be solved individually or in small groups in breakout rooms. + +## Type along +- You will be able to follow along using the built-in terminals in all OSs. + - Linux/Mac Terminal + - Windows: Powershell or command prompt +- For your day-to-day work you may benefit from other applications, see the [Login session](https://uppmax.github.io/uppmax_intro/login2.html) + +## The topics +### Linux I at 9-12 Monday +- Overview +- Logging in to UPPMAX +- Linux basics +- Using the command line + +### Intro to UPPMAX 13-16 Monday +- Organisation +- HPC computing +- SNIC +- Using software and tools +- Submitting jobs to calculation nodes (intro) +- Bianca cluster + +### Linux II 9-12 Tuesday +- Using the command line level 2 +- Input, output and pipes +- Wildcards +- The find command +- Links +### Linux III 13-16 Tuesday +- The grep command +- Using the command line level 3 +- Variables, parameters and $(...) +- Filenames +### Bash Scripts 9-12 Wednesday +- Bash scripting, why's and basics +- Parameters +- The IF statement +- File tests +- FOR and WHILE loops +- Background processes and job control +### Slurm at UPPMAX 13-16 Wednesday +- A workload manager +- Queuing +- Monitoring +- in queue +- while running +- when finished +- Job testing within the interactive or "devel" mode +- Scripting +### Python I 9-12 Thursday +- Introduction +- Basics +- Data types +- Control structures +- Writing and executing a program +### Python II 13-16 Thursday +- Creating a program +- Functions + +## Today + +### Preliminary schedule +- First time in this manner so timings are preliminary. + +**Schedule** +- 9.00 Intro +- 9.15 Overview +- 9.30 Log in +- 9.55 15 min coffee break +- 10.10 Linux intro +- 10.40 Linux basic commands +- (c. 11.00 10 min break) +- 11.30 Overview - part 2 +- 11.35 HPC +- 11.40 NAISS and projects +- 11.50 Q/A if time allows +- 12.00 **LUNCH** +- 13.00 Software and tools +- 13.30 Using compute nodes +- (c. 14.00 10 min break) +- 14.30 Working on UPPMAX + - Room for sharing tips + - Then run to coffee break +- (14.45 NAISS-SENS and Bianca (Do we have any users here?)) +- 14.55 15 min coffee break +- 15.10 NAISS-SENS and Bianca p2 +- 15.30 Courses and workshops +- 15.35 Summary +- 15.40 Q/A + +### The New documentation + +- New look later in spring for the official UPPMAX webpage, now at +- We like the documentation/tutorials in another style: + - boxes, graphs, programming code style +- Let's look at the documentation! + - Getting started + - Cluster user guide + - Software + - Databases + +### Style of todays lectures + - Information sessions + - Case sessions with usage of the new documentation + +## Icebreaker in HackMD +- [https://hackmd.io/@UPPMAX/Intro-Course-QaA](https://hackmd.io/@UPPMAX/Intro-Course-QaA) + diff --git a/_sources/linux.md.txt b/_sources/linux.md.txt new file mode 100644 index 0000000..59fbc84 --- /dev/null +++ b/_sources/linux.md.txt @@ -0,0 +1,261 @@ +# Linux +- The "operating system" of the UPPMAX and most of the other clusters is **Linux**. + +```{questions} +- What is Linux? +- How to use the command line? + +``` + +```{objectives} +- We'll briefly get an overview of Linux + - How the command line works + - Some text editors + - Things to be aware of +``` + +## What is Linux? + +![Content](./img/pingvin.png) + +- Daily speaking: The Linux Operating system is a UNIX like and UNIX compatible Operating system. +- Linux is a "Kernel" on which many different programs can run. +- The shell (bash, sh, ksh, csh, tcsh and many more) is one such program. + +![Content](./img/images.jfif) + + + +- Actually, for it to be an OS, it is supplied with GNU software and other additions giving us the name **GNU/Linux**. + - [Linux naming controversy]((https://en.wikipedia.org/wiki/GNU/Linux_naming_controversy) + + + +![Content](./img/gnu.png) + + + +- Linux has a multiuser platform at its base which means permissions and security comes easy. + + +### Linux comes in different distributions, dialects or, say, flavours. +- UPPMAX runs CentOS and RedHat + + + +![Content](./img/flavours.png) + + + +## Using the command line + +### Command line with bash (Bourne Again Shell) +- A Unix shell and command language. +- Often default shell + +![Content](./img/shell.jpg) + +- The command-line interface: the bash prompt $ +- bash can be seen as a program that finds and runs other programs +- bash is scripting language that is referred to as a shell + - (because it sits around the kernel making it easy to interact with) + + ![Content](./img/unix_architecture.jpg) + + +### The prompt + +[info]$ program word1 word2 word3 […] + +- [info] is configurable, and usually tells you who you are, on what system, and where in the file system. + + - Example: + + ```bash= + [bjornc@rackham3 linux_tutorial]$ + ``` + + - For changing info (only for advanced users!) Does not matter for this course!: + - + - The program to run is the first word + - All words are separated by spaces + +![folders](./img/folders.png) + +### Example bash command + +
+ +```{image} ./img/mv_inbox.png +:alt: mv inbox +:width: 300px +:align: left +``` + +

+ +```{image} ./img/program_flags.png +:alt: program flags +:width: 300px +:align: left +``` + +

















+ + +
+ + + +- Terminal screen shows + +

+ +![Content](./img/screen.png) + + +### Tab Completion +
+ +```{image} ./img/tab.png +:alt: tab +:width: 200px +:align: left +``` + +




+
+ +- Whenever you’re writing a path or filename on the bash prompt, you can strike the ‘tab’ key to ask Bash to complete what you’re writing. + +- Get in the habit of this — it will save you many hours! + + +## Editing files with file/text editors + +![Content](./img/edit.png) + + +### gedit +- graphical user interface — GUI, needs X-server + +- Also graphical editor within MobaXterm + +### nano +- keyboard shortcuts shown on-screen) +- Cheatsheet: [http://staffwww.fullcoll.edu/sedwards/nano/UsefulNanoKeyCommands.html](http://staffwww.fullcoll.edu/sedwards/nano/UsefulNanoKeyCommands.html) + - ^ = Ctrl, M = meta key) + - Windows M = Alt + - On Mac: in the Terminal.app go to Preferences -> Settings -> Keyboard and turn on "Use option as meta key": then M = Alt + + +````{solution} Not to try today if you haven't used before!!! + +```{warning} +- we suggest that you learn this tools before trying to work with them on UPPMAX +- If you start one of these editors you may have difficulties to exit! +``` + +**vim** +- fast and powerful, once you learn it +- on UPPMAX started with command ``vi`` + + - 1. Insert mode (type like normal text editor. Press ``i`` for insert mode) + - 2. Command mode (give commands to the editor to get things done . Press `` for command mode) + - Cheat sheet: [https://coderwall.com/p/adv71w/basic-vim-commands-for-getting-started](https://coderwall.com/p/adv71w/basic-vim-commands-for-getting-started) + +**gvim** +- ``vi``m with a GUI, lots of features very Fast + +**emacs** +- fast and powerful, once you learn it + - Cheat sheet: [https://www.gnu.org/software/emacs/refcards/pdf/refcard.pdf](https://www.gnu.org/software/emacs/refcards/pdf/refcard.pdf) + - (C = ``) +- also With GUI + - ``emacs –nw`` + - keeps you in terminal window. +```` + +## The graphical editors +- When starting the graphical versions of an editor, add ``&`` to be able to use the command line while program is open. + - Ex: `gedit &` + - If not, you can `+z` and type `bg` to send program to background. + +```{exercise} Try gedit + - start gedit with ``gedit &`` + - write something + - save +``` + +```[tip} +- ``ctrl-C`` interupts a program or a command that is "stuck" +- ``ctrl-Z`` pauses a program, can be continues in background (``bg``) or foreground (``fg``) +- ``ctrl-D`` quits some programs +``` + +## Typical sources of error + +![Content](./img/cross.png) + +```{Warning} +- Capitalization matters in file names and program names +- Spaces matter. + - Always have a space after the program name. + - Don’t add spaces within file names. +- Check that you are in the right place in the file system. +- File permissions. Check that the right read, write and execute permission are set. See next session. +``` + + + +## Caution!! + +![Content](./img/caution.png) + + +```{Warning} + +- There is no undo for: + - copy (`cp`), + - move (`mv`), and + - remove (`rm`). +- **Beware of overwriting files and deleting the wrong ones.** +``` + +```{Note} +- **Tip: make "`rm`" ask if you really want to erase:** + - Within a session: Type in the command prompt + + alias rm='rm -i' + + - Override asking with + + rm –f <> + + - Edit file `.bashrc` in `home` directory by adding the alias line for this to start everytime. +- This will also work for ``mv`` and ``cp``! +``` + +```{Note} +- If you do destroy your data, email UPPMAX support, we may be able to help. +``` + + ```{keypoints} +- Linux Operating system is a UNIX-like and UNIX compatible Operating system. +- Typical command: + $ program word1 word2 word3 […] +- Example of file editors + - terminal + - nano + - vim + - emacs + - graphical: + - gedit +- Tips + - use Tab completion + - capitalization and spaces matters + - no undo:s for copying, moving and removing + - Solution: ``alias rm='rm -i' `` +``` + + diff --git a/_sources/linux_basics.md.txt b/_sources/linux_basics.md.txt new file mode 100644 index 0000000..c838171 --- /dev/null +++ b/_sources/linux_basics.md.txt @@ -0,0 +1,572 @@ +# Basic toolkit + + +![Caption](./img/terminal.png) + + +```{objectives} +- Let's dig into the most important BASH commands +- We'll do a type-along session +``` + +## We will cover these commands +### Navigation and file management + +1. `pwd`   present directory +1. `ls`  list content +1. `cd`  change directory +1. `mkdir`  make directory +1. `cp`  copy +1. `scp`  securely remotely copy +1. `mv`  move +1. `rm`  remove +1. `rmdir`  remove empty directory + +### Read files and change file properties + +10. `cat`  print content on screen +11. `head`  print first part +12. `tail`  print last part +13. `less`  browse content +14. `tar`  compress or extract file +15. `chmod`  change file permissions +16. `man`  info about a command + +## File system Navigation +### pwd — where are you now? “Print name of current/Working Directory” + +``` {code-block} console +$ pwd + +$ pwd -P +``` +- ``-P`` gives you the physical path, + - ignores how you got there + + +### ls — list directory contents + +Type ``ls`` to display the contents of the current directory. + +``` {code-block} console +$ ls -a +``` + +``-a`` also shows hidden files and directories. + +``` {code-block} console +$ ls -l +``` +``-l`` gives you listed and detailed information in alpha-numerical order + +```{tip} +- the short-hand command ``ll`` will make a list as above, but highlight different file types with different colors +``` + +``` {code-block} console +$ ls -lt +``` + +``-lt`` sorts things by time modified. + +``` {code-block} console +$ ls –lrt +``` + +``-r`` gives reversed order, so in this case newest in last line. + +``` {code-block} console +$ man ls +``` + +- for complete information about a command. +- TIP: `-$ man ` works for almost any command! + - scroll with arrows and quit with ``q``. + + +### cd — Change the shell working Directory +- To change directory, use ``cd `` + +``` {code-block} console +$ cd /proj/introtouppmax + +$ pwd + +$ ls + +$ cd labs + +$ pwd +``` + +````{challenge} Experiment with cd +- Experiment with ``cd``. +- Try adding ```` or extra ``/`` in various places. +- Use *tab completion* to avoid typos and typing ``ls`` a lot. + +- Figure out the use of the following: + +``` {code-block} console +$ cd - + +$ cd .. + +$ cd + +$ cd ~ + +``` +```` + +````{solution} Solution +- `cd -` : goes back to your last directory + +- `cd ..` : goes a level up in the hierarchy + +- `cd` : goes to home directory + +- `cd ~` : also goes to home directory +```` + + + +## Copy, Create, Move +### mkdir — make directories + +```{warning} +- Make sure you’re in your home directory by `cd ~` +``` + +- Create a new directory ``uppmax-intro`` + +``` {code-block} console +$ cd ~ +$ mkdir uppmax-intro +``` +- Go in there: + +``` {code-block} console + $ cd uppmax-intro/ +``` + + +### cp — copy files and directories + +- Copy files with: `cp ` +- Set target to ``.`` to keep name and to point at present directory. + +``` {code-block} console +$ cp /proj/introtouppmax/labs/linux_tutorial/ . +``` + +- Well, that didn’t work. What does the error say? +- So... try +``` {code-block} console +$ cp -r /proj/introtouppmax/labs/linux_tutorial/ . +``` + +``-r`` is for recursive, meaning including files and subdirectories! + +- Move to your just created ``linux_tutorial/`` + +``` {code-block} console +$ cd linux_tutorial + ``` +- Make a copy of the file “newfile” in the same directory: +``` {code-block} console +$ cp newfile copyfile +``` + +### scp —secure copy (remote file copy program) + +- Linux/MacOS: To copy data to/from Rackham, you can use ``scp`` from the terminal on your local machine: +- Upload from present directory on local machine to your home directory on cluster. + - Example (*not done by us today*) + +``` {code-block} console +[bob@macbook]$ scp myinput bob@rackham.uppmax.uu.se:~/copyofmyinput + +[bob@macbook]$ scp myinput bob@rackham.uppmax.uu.se:~/ # (keeping filename) +``` + +- Download +``` {code-block} console +[bob@macbook]$ scp bob@rackham.uppmax.uu.se:~/mydata copyofmydata + +[bob@macbook]$ scp bob@rackham.uppmax.uu.se:~/mydata . # (keeping file name) +``` + +### mv — move/rename file + +- Moving files works just like copying files: +- `mv ` +- Move the copy you just made to another place: +``` {code-block} console +$ mv copyfile ../ +``` +- Rename it. +``` {code-block} console +$ mv ../copyfile ../renamedfile +``` + +## Archiving +**tar — archiving and compression** + +- We’re going to need more files. Let's extract the tar.gz file (tared and gzipped file) + +``` {code-block} console +$ tar -vxzf files.tar.gz +``` +- The flags mean: + - verbosely + - extract + - gzipped + - filename +- Order of flags may matter! + - ``f`` should be in the start or in the end! +- You should see a list of files being extracted + +````{tip} +- To compress use the flag `-c`instead of `-x` + +``` {code-block} console +$ tar -czfv +``` +```` + +## Deleting +### rm — delete files or directories + +```{Note} +- **Tip: make "rm" ask if you really want to erase:** + - Within a session: Type in the command prompt + + alias rm='rm -i' + + - Override asking with + + rm –f <> + + - Edit file ".BASHrc" in /home directory by adding the alias line for this to start every time. +- This will also work for mv and cp! +``` + +- Deleting files works just like copying or moving them: +`rm ` + +- Try it out: +``` {code-block} console +$ rm ../renamedfile + +$ rm this_is_empty + ``` + +- hmmmm... + + +### rmdir — delete an empty directory + +- We need another command to delete directories + +``` {code-block} console +$ rmdir this_is_empty + +$ rmdir this_has_a_file +``` +- Again?? + +- Is there a way to use rm to delete directories? + +````{solution} +- Recursive commands `-r` are applied to directories and their contents +``` {code-block} console +$ rm -r this_has_a_file +``` +```` + +## Help +### man — manual, look up the right flags + +- Nobody can remember whether it’s ``-R`` or `-r` for recursive, or if ``-f`` lets you choose a file or forces an action. + +``` {code-block} console +$ man ls +``` +- shows you how to use ``ls`` and all its options +- Type `/` to search for a keyword, use `n` (forward) and ´N` (backward) to scan through hits. +- Scroll with arrows. +- Type `q` to quit. + +```{challenge} +- Spend some time now to browse the man pages for the commands you’ve just learned! +``` + + + +## Let’s get wild with Wildcards +![Caption](./img/wildcards_bear.png) + +``` {code-block} console +$ ls many_files + +$ ls many_files/*.txt + +$ ls many_files/file_1*1.docx + ``` + +- Want to clean out temporary files ending in .tmp in all the subdirectories? + +```{warning} +- NB! It could be wise to do `ls -a */*.tmp` first to see what will be deleted... +``` +``` {code-block} console + $ rm */*.tmp +``` + +```{challenge} Exercise +- Exercise: Create a new directory and move all .txt files in many_files to it. +``` + +## Reading files + +- In Linux, you can (if you wish) also display files without being able to change them + +``` {code-block} console +$ cd old_project + +$ ls +``` + +- Hmm, which of these files are useful? + + +### cat - concatenate files and print on the standard output + +![Caption](./img/cat.png) + +- ``cat`` dumps the contents of files to the terminal as text + +``` {code-block} console + $ cat the_best +``` + +- Yummy! + +``` {code-block} console + $ cat a +``` +- What's this???? + +- **Concatenate** files with this wizardry: + +``` {code-block} console + $ cat a the_best > combinedfiles.txt +``` +- File ``a`` is written first and ``the_best`` is appended + +### head — display the top (heading) of a file + + +![Caption](./img/head.png) + +``` {code-block} console + $ head a + ``` +- You can choose how many lines to display (default 10) + +``` {code-block} console + $ head -n 4 a + ``` + + +### tail — display the end of a file + + +![Caption](./img/tail.png) + +- Tail is the same as head, but for the other end. + +``` {code-block} console + $ tail -n 5 a +``` + +- Handy to look at log files or to figure out the structure of a text file. + + +### less — read a whole file + +- cat doesn’t really work for long files + +``` {code-block} console + $ less a +``` + +- Search with `/` and `n`/`N` +- Hit `q` to quit. +- scroll with arrows. +- `man` uses `less`! + + “less is more” + +## File permissions + + +![Caption](./img/permission.png) + +### Example + +``` {code-block} console +$ ls -l + + drwxrwxr-x 2 marcusl marcusl 4096 Sep 19 2012 external_hdd + -rwxr-xr-x 1 marcusl marcusl 17198 Jul 16 14:12 files.tar.gz +``` +- Leading symbol: + - `d` directory + - `-` regular file + - `l` symbolic link (more on this tomorrow) + - Others exist, but you can ignore them for now + + ``` {code-block} console + $ ls -l + + drwxrwxr-x 2 marcusl marcusl 4096 Sep 19 2012 external_hdd + + -rwxr-xr-x 1 marcusl marcusl 17198 Jul 16 14:12 files.tar.gz + ``` + +- Three sets of “rwx” permissions + - rwx: r ead, w rite, ex ecute + - User: the user account that owns the file (usually the one that created it) + - Group: the group that owns the file (usually the *project group* in /proj/xyz or the user’s group elsewhere) + - Others: everyone else on the system (literally a thousand strangers) + +- r – read + - Files: Read the contents of the file + - Directories: List the files in the directory + +- w – write + - Files: Modify the file + - Directories: Add, rename, or delete files in the directory + +- x – execute + - Files: Run the file as a program + - Directories: Traverse the directory (e.g. with “cd”) + +``````{solution} For the interested + - Now try: + + ``` {code-block} console + $ ls -l /proj/introtouppmax/ + ``` + +- Huh, ``rwxrwsr-x``? +- `s` in the group means `x` but with *gid bit set* ( g roup id of creator not launcher). + - The s or sticky bit is a group permission on directories which changes the default behaviour of new files are created with the same group_id as the users group_id to new files inheriting the group_id from the parent directory. +- `S` means `-` with gid bit set (rarely seen). +- Among other things, this makes the default group for new files/subdirectories the, for instance, ``p_introtouppmax`` group. + +`````` + +## Changing permissions +**chmod** — change file mode bits + +**If you own, i.e. created, the file or directory, you can modify the content** + +```{admonition} Common issues +- Files with `w` can be modified and destroyed by accident. Protect your data! +- If you want to share data or scripts with a person not in your project (e.g. support staff like me), you can! +- If you want to keep non-members from even seeing which files you have, you can! +``` + +### Syntax + +`chmod ` + +- `` is of the form: For whom, Modify, What permission(s) +- For whom? + - `u`: user/owner + - `g`: group, often the members to a certain project + - `o`: others + - `a`: all + - if not set changes are applied for user AND group +- Modify? + - `+`: add permissions, + - `-`: remove + - `=`: set equal to + - `=` usually causes unmentioned bits to be removed except that a directory's unmentioned set user and group ID bits are not affected. +- What permissions? + - `r`, `w`, `x`, i.e. the actual permission + +#### Examples + +- `` can be e.g.: + - `u+x` : lets You (owner) run a script you just wrote + - `-w` : no write permissions for **owner+group** + - warning: if `w` was already set for *others* it will be kept!! + - `+rw` : let user and group members read and edit this file, not others if not already set + - `=xw` : let group members go into your directory and put files there, but not see which files are there, others are not affected + - `a=xw` : set xw for everyone + +- chmod takes flags as usual, e.g. + - `-R` for recursive (i.e. all files and sub-directories therein) + +``````{solution} chmod 755 style - binary sum - "octal bit mask” + +- Online, you will come across e.g. `chmod 755 `. What does this mean? It’s an "octal bit mask”: + +- Each digit corresponds to the **binary sum** for the *owner*, *group* and *others*, respectively. + + - ``7 = 4 + 2 + 1 = r + w + x`` All permissions + - ``5 = 4 + 0 + 1 = r + + x`` Read and execute permission + +- 755 then means all permissions for owner, but limiting write permissions for the group and all others + +- What number would `rw` be? + +```{solution} +6 + ``` +`````` + +```{challenge} chmod — Hands-on + +- In your *locally created* ``linux_tutorial`` directory, find important files and old saved data that you wouldn’t want to lose (*imagine*). + - Directories: important_results/, old_project/ + - File: last_years_data +- Use chmod to remove write permission from those files and directories (use the `-R` flag (not `-r`) to also do the files in the directories). + - Take a moment to play around with chmod and explore the effects of permissions on files and directories. +``` + +``````{solution} +``` {code-block} console +$ chmod -wR + ``` +`````` + +**More about BASH command line and scripts on Tuesday and Wednesday!** diff --git a/_sources/linux_cases.md.txt b/_sources/linux_cases.md.txt new file mode 100644 index 0000000..2af88f2 --- /dev/null +++ b/_sources/linux_cases.md.txt @@ -0,0 +1,159 @@ +# Linux and basic toolkit + +- The "operating system" of the UPPMAX and most of the other clusters is **Linux**. + +```{questions} +- What is Linux? +- How to use the command line? +``` + +```{objectives} +- We'll briefly get an overview of Linux + - How the command line works + - Some text editors + - Things to be aware of +``` + +```{instructor-note} +- Approx timing: 10:10-10:40 +- Theory and type-along +``` + +```{admonition} See the documentation +- +``` + +```{tip} +- TAB complete: + - Whenever you’re writing a path or filename on the bash prompt, you can strike the ‘tab’ key to ask Bash to complete what you’re writing. + - Get in the habit of this — it will save you many hours! + +**These commands are useful in the command line when something is stuck or a program is limiting you to do further work.** +- ``ctrl-C`` interupts a program or a command that is "stuck" +- ``ctrl-Z`` pauses a program, can be continues in background (``bg``) or foreground (``fg``) +- ``ctrl-D`` quits some programs +``` + + +```{Warning} + +- There is no undo for: + - copy (`cp`), + - move (`mv`), and + - remove (`rm`). +- **Beware of overwriting files and deleting the wrong ones.** +``` + +```{Note} +- **Tip: make "`rm`" ask if you really want to erase:** + - Within a session: Type in the command prompt + + alias rm='rm -i' + + - Override asking with + + rm –f <> + + - Edit file `.bashrc` in `home` directory by adding the alias line for this to start everytime. +- This will also work for ``mv`` and ``cp``! +``` + + +```{Note} +- If you do destroy your data, email UPPMAX support, we may be able to help. +``` + + ```{keypoints} +- Linux Operating system is a UNIX-like and UNIX compatible Operating system. +- Typical command: + $ program word1 word2 word3 […] +- Example of file editors + - terminal + - nano + - vim + - emacs + - graphical: + - gedit +- Tips + - use Tab completion + - capitalization and spaces matters + - no undo:s for copying, moving and removing + - Solution: ``alias rm='rm -i' `` +``` + + + +## Basic toolkit + +```{objectives} +- Let's dig into the most important BASH commands +- We'll do a type-along session +``` + +```{instructor-note} +- Approx timing: 10:40-11.30 +- Type-alongs +``` + +## We will cover these commands +### Navigation and file management + +1. `pwd`   present directory +1. `ls`  list content +1. `cd`  change directory +1. `mkdir`  make directory +1. `cp`  copy +1. `scp`  securely remotely copy +1. `mv`  move +1. `rm`  remove +1. `rmdir`  remove empty directory + +### Read files and change file properties + +10. `cat`  print content on screen +11. `head`  print first part +12. `tail`  print last part +13. `less`  browse content +14. `tar`  compress or extract file +15. `chmod`  change file permissions +16. `man`  info about a command + + +```{admonition} See the documentation +- +``` + +```{challenge} chmod — Hands-on + +- In your *locally created* ``linux_tutorial`` directory, find important files and old saved data that you wouldn’t want to lose (*imagine*). + - Directories: important_results/, old_project/ + - File: last_years_data +- Use chmod to remove write permission from those files and directories (use the `-R` flag (not `-r`) to also do the files in the directories). + - Take a moment to play around with chmod and explore the effects of permissions on files and directories. +``` + +``````{solution} +``` {code-block} console +$ chmod -wR +``` +`````` + +``````{solution} For the interested + +- Now try: + +```console +$ ls -l /proj/introtouppmax/ +``` + +- Huh, ``rwxrwsr-x``? +- `s` in the group means `x` but with *gid bit set* ( g roup id of creator not launcher). +- The s or sticky bit is a group permission on directories which changes the default behaviour of new files are created with the same group_id as the users group_id to new files inheriting the group_id from the parent directory. +- `S` means `-` with gid bit set (rarely seen). +- Among other things, this makes the default group for new files/subdirectories the, for instance, ``p_introtouppmax`` group. +`````` + + +**More about BASH command line and scripts on Tuesday and Wednesday!** + +- https://www.uppmax.uu.se/support/courses-and-workshops/uppmax-introductory-course/ diff --git a/_sources/login.md.txt b/_sources/login.md.txt new file mode 100644 index 0000000..f76286c --- /dev/null +++ b/_sources/login.md.txt @@ -0,0 +1,181 @@ +# Login + +```{questions} +- How to reach UPPMAX clusters? +- Can one use graphics? +- Where do I arrive when I log in? + +``` + +```{objectives} +- We'll relate our login session to the specific "area" in the cluster. +- We'll go through platform specific (Mac/Linux/Windows) ways to log in to UPPMAX. +- Try yourself! +``` + +```{note} +If you lack a user account, visit the [Getting started page](https://www.uppmax.uu.se/support/getting-started/course-projects/) +``` + +## The login +- When logging in to UPPMAX from your local computer you will arrive to your home folder at the login node. +- This means that only light analysis and and calculations should be made here. +- You will see this in the prompt after "@" as the clustername and a low number. For instance: + ```bash= + [@rackham3 linux_tutorial]$ + ``` +- You willl ater learn how to reach the calculation nodes. Then the prompt states the node number with a single letter, like "r" for Rackham. For instance: + ```bash= + [@r484 linux_tutorial] + ``` + + +## MAC and LINUX users + +### Terminal + +- Download XQuartz or other X11 server for Mac OS + - [https://www.xquartz.org/](https://www.xquartz.org/) + - This is to enable graphics. + +- Start built-in Terminal. + +```bash= +$ ssh -Y @rackham.uppmax.uu.se +``` +- "< >" prompts you to set the keyword specific for you or your needs. In the example above, this is basically your username. + +![Terminal](./img/Mac_terminal.png) + +### Graphical file manager + +- For copying of files with sftp (secure file transfer protocol) between your client computer (where you are) and the cluster **Filezilla** can be the choice. +- [https://filezilla-project.org/download.php?type=client](https://filezilla-project.org/download.php?type=client) + + +![Caption](./img/fz3_osx_main.png ) + +```{callout} Problems with installations? +The built-in terminal without X11 is sufficient first days of the course! +``` + +## Windows users + +### Most straight-forward +- Install a ssh (secure shell) program with built-in X11 and sftp file manager + - [MobaXterm](https://mobaxterm.mobatek.net/) + - sftp frame makes it easy to move, upload and download files. + - See section below. + +- If problem with the above installation, the putty.exe terminal is sufficient first days of the course! + + - No graphics. + + - [https://www.putty.org/](https://www.putty.org/) + +![Caption](./img/putty.jpg) + + + + +### MobaXterm + +![Caption](./img/mobax.jpg ) + +- Start local terminal and a SSH session by: + +```bash= +$ ssh -Y @rackham.uppmax.uu.se +``` + +![Caption](./img/mobax_start1.jpg) + +- Or even better, create and save a SSH session, as shown in image below. + - This allows you to use MobaXterm as a file manager and to use the built-in graphical texteditor. + - You can rename the session in the Bookmark settings tab. + +![Caption](./img/mobax_start.jpg) + + +### Graphical file manager (Windows) + +- For copying of files between your client computer (where you are) and the cluster WinSCP can also be the choice. + - [https://winscp.net/eng/download.php](https://winscp.net/eng/download.php) + +```{callout} Problems with installations? +The built-in terminal without X11 is sufficient first days of the course! +``` + + +## X11-forwarding from the command line (generally) + +- Graphics can be sent through the SSH connection you’re using to connect + - Use primarily `ssh -Y <...>` or secondary `ssh -X <...>` + +- The X servers that enables graphics are needed, as mentioned above! + - When starting a graphical program, a new window will open, but your terminal will be “locked”. + - Run using "`&`" at the end to run it as a background process e.g. "`xeyes &`" or “`gedit &`” + +![Caption](./img/xeyes.png) + +- Alternatively, use `-z` to put e.g. gedit to sleep and type "`bg`" to make last process in background. + +``````{challenge} Login to **Rackham**, using your terminal +- First try: +```bash= +$ ssh -Y @rackham.uppmax.uu.se +``` +- If you receive errors or warnings, instead try: +```bash= +$ ssh @rackham.uppmax.uu.se +``` +- If you do have X11 installed: +```bash= +$ xeyes & +``` +`````` + +## ThinLinc (all platforms!) + +- Both Rackham and Bianca offer graphical login. +- This gives you a desktop environment, as if you were working on your own computer! +- On web: + - [https://rackham-gui.uppmax.uu.se](https://rackham-gui.uppmax.uu.se) + - [https://bianca.uppmax.uu.se](https://bianca.uppmax.uu.se) + - requires [2-factor authentication](https://www.uppmax.uu.se/support/user-guides/setting-up-two-factor-authentication/) + +![Caption](./img/Thinlinc2.jpg) + + - Or use the client (only for Rackham) + - [https://www.cendio.com/thinlinc/download](https://www.cendio.com/thinlinc/download) + +```{challenge} Try Thinlinc from web + - Try the web version now if you don't already have the software installed! + ``` + + ```{keypoints} +- You reach UPPMAX clusters either using a terminal or Thinlinc +- Graphics are included in Thinlinc and from terminal if you have enabled X11 +- When you log in from your local computer you will always arrive at a login node with limited resources. + - You reach the calculations nodes from within the login node (See Submitting jobs section) +``` diff --git a/_sources/login2.md.txt b/_sources/login2.md.txt new file mode 100644 index 0000000..17077a4 --- /dev/null +++ b/_sources/login2.md.txt @@ -0,0 +1,135 @@ +# Login + +```{questions} +- How to reach UPPMAX clusters? +- Where do I arrive when I log in? Login or calculation node? +- What clients should/could I use for my work at UPPMAX? +- How can I enable graphics? + +``` + +```{objectives} +- We'll go through platform specific (Mac/Linux/Windows) ways to log in to UPPMAX +- See different clients +- Enable graphics +- Try yourself! +``` + +```{instructor-note} +- Approx timing: 9:30-9:55 +- Demos/type-alongs +``` + +```{note} +If you lack a user account, visit the [Getting started page](https://www.uppmax.uu.se/support/getting-started/course-projects/) +``` + +## General understanding + +- When logging in to UPPMAX from your local computer you will arrive to your home folder at the login node. +- This means that only light analysis and and calculations should be made here. +- You will see this in the prompt after "@" as the clustername and a low number. For instance: + ```console + [@rackham3 linux_tutorial]$ + ``` +- You will later learn how to reach the calculation nodes. Then the prompt states the node number with a single letter, like "r" for Rackham. For instance: + ```console + [@r484 linux_tutorial] + ``` + + +## The login + +```{discussion} Login procedure + **Which login procedure is best for You, depends on:** + - Your background + - Your OS environment, + - Your planned interaction with your local computer + - Your planned use of graphics on the cluster +``` + +- We will use Rackham for the course + +```{admonition} See the documentation +- [Get inside SUNET](http://docs.uppmax.uu.se/getting_started/get_inside_sunet/) +- [Log in to Rackham](http://docs.uppmax.uu.se/getting_started/login_rackham/) +``` + +## Exercises + +```{challenge} Log in with a ternminal + - Use any tool that you have installed +``` + +```{challenge} Log in with a ternminal but enable graphics +- Use any tool that you have installed +``` +```{solution} +- First try: +```console +$ ssh -Y @rackham.uppmax.uu.se +``` +- If you receive errors or warnings, instead try: +```console +$ ssh @rackham.uppmax.uu.se +``` +- If you do have X11 installed: +```console +$ xeyes & +``` + +```{challenge} Try Thinlinc from web +- Try the web version now if you don't already have the software installed! +- +``` + +```{admonition} Login procedure + If you plan to: + - do **day-to-day** work where *terminal shell is sufficient* + - Mac: *Terminal, iTerm2* + - Linux: *Terminal* + - Windows: *Putty*, *Windows Powershell* or even *command prompt (CMD)* + + + - **interact with your local computer** + - Mac/Linux: you can always work in a local shell (mutiple terminal windows open) + - (S)FTP browser: *Filezilla*, *Cyberduck* + - Windows + - (S)FTP browser: *WinSCP* + - *MobaXterm* has built-in SFTP browser + - you may benefit from having a *Windows Subsystem for Linux, WSL(2)* + + + - **do day-to-day work** with **some graphical applications (X forwarding)** + - Mac: *Terminal, iTerm2 + XQuartz* + - Linux: *Terminal* + - Windows: *MobaXterm* + + + - integrate you cluster work with **code development** + - All OS: Example *Visual Studio Code* + + + - use **sophistic graphical interfaces** like *RStudio and MATLAB* etcetera + - *ThinLinc application* + + + - use **Bianca** + - *ThinLinc from web* + +``` + + + ```{keypoints} +- When you log in from your local computer you will always arrive at a login node with limited resources. + - You reach the calculations nodes from within the login node (See Submitting jobs section) +- You reach UPPMAX clusters either using a terminal client or Thinlinc +- Graphics are included in Thinlinc and from terminal if you have enabled X11. +- Which client to use? + - Graphics and easy to use + - ThinLinc + - Best integrated systems + - Visual Studio Code has several extensions (remote, SCP, programming IDE:s) + - Windows: MobaXterm is somewhat easier to use. + +``` diff --git a/_sources/naiss.md.txt b/_sources/naiss.md.txt new file mode 100644 index 0000000..8d26491 --- /dev/null +++ b/_sources/naiss.md.txt @@ -0,0 +1,14 @@ +# NAISS and projects +```{objectives} +- Let's step up and have an overview of NAISS +``` + +```{instructor-note} +- Approx timing: 11.40-11:50 +- Information +``` + +```{admonition} See the documentation +- +``` + diff --git a/_sources/overview.md.txt b/_sources/overview.md.txt new file mode 100644 index 0000000..3a79715 --- /dev/null +++ b/_sources/overview.md.txt @@ -0,0 +1,115 @@ +# Overview + +```{questions} +- What is UPPMAX? +- What is SNIC/NAISS? +- What is a cluster? + +``` + +```{objectives} +- We'll get an overview of UPPMAX and SNIC/NAISS and how a computer cluster works +``` + +```{instructor-note} +- Approx timing: 9:15-9:30 +- Theory +``` + +**UPPMAX = UppMACS** +- [Uppsala Multidisciplinary Center for Advanced Computational Science](http://uppmax.uu.se) + +## NAISS +- [**National Academic Infrastructure for Supercomputing in Sweden**](https://www.naiss.se/) +- Mission: to provide a quality high-performance computing environment nationally + +- Starting 1 January 2023, the National Academic Infrastructure for Supercomputing in Sweden (NAISS) is the new organization for high-performance computing, storage, and date services for academic users in Sweden. + - Replaces SNIC — the Swedish National Infrastructure for Computing + - Linköping university is the host organization for NAISS, but NAISS acts independently with a national perspective and responsibility. +- From the users perspective, there will initially only be minimal differences between the SNIC and NAISS regimes. + - FAQ:s — +- Application rounds: +- NAISS and Uppsala University fund UPPMAX — UU’s supercomputing center. + + +## UPPMAX missions +- Runs the clusters placed in Uppsala. +- More details in the afternoon about Organisational orienteering! + + +## UPPMAX systems + +- Clusters + - Rackham (general purpose) + - Snowy (Long runs and GPU:s) + - Bianca (sensitive data) + - Miarka (NGI production) + - genomic data from the sequencing machines are pre-processed before they are delivered to the researcher. +- Storage + - On-load, directly connected to the clusters + - Off-load, for large data not needed for computation analysis anymore +- Cloud + - Dis (region EAST-1) + +## High Performance Computing — HPC +### What is a cluster? + +- A network of computers, each computer working as a **node**. + +- From small scale RaspberryPi cluster... + +![RaspBerry](./img/IMG_5111.jpeg) + +- To supercomputers like Rackham. + +![Rackham](./img/uppmax-light2.jpg) + +- Each node contains several processor cores and RAM and a local disk called scratch. + +![Node](./img/node.png) + +- The user logs in to **login nodes** via Internet through ssh or ThinLinc. + + - Here the file management and lighter data analysis can be performed. + +![RaspBerry](./img/nodes.png) + +![RaspBerry](./img/Bild1.png) + +- The **calculation nodes** have to be used for intense computing. + +## Overview of the UPPMAX systems + +```{mermaid} + graph TB + + Node1 -- interactive --> SubGraph2Flow + Node1 -- sbatch --> SubGraph2Flow + subgraph "Snowy" + SubGraph2Flow(calculation nodes) + end + + thinlinc -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + terminal -- usr --> Node1 + terminal -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + Node1 -- usr-sensXXX + 2FA + no VPN ----> SubGraph1Flow + + subgraph "Bianca" + SubGraph1Flow(Bianca login) -- usr+passwd --> private(private cluster) + private -- interactive --> calcB(calculation nodes) + private -- sbatch --> calcB + end + + subgraph "Rackham" + Node1[Login] -- interactive --> Node2[calculation nodes] + Node1 -- sbatch --> Node2 + end +``` + + +```{keypoints} +- NAISS makes available large-scale high-performance computing resources, storage capacity, and advanced user support, for Swedish research. +- UPPMAX runs the local resources placed at Uppsala Universtiy +- A cluster consists of several inter-connected computers that can work individually or together. +``` + diff --git a/_sources/overview2.md.txt b/_sources/overview2.md.txt new file mode 100644 index 0000000..7a3c59b --- /dev/null +++ b/_sources/overview2.md.txt @@ -0,0 +1,50 @@ +# Overview + +```{objectives} +- Let's recap and go a little deeper into the organization of NAISS/UPPMAX +``` + +```{instructor-note} +- Approx timing: 11.30-11:35 +- Theory +``` + +## Organisational orienteering + +- [VR/SRC](https://www.vr.se/english.html) and a consortium of universities funds [NAISS](http://www.naiss.se) — the National Academic Infrastructure for Supercomputing in Sweden +- NAISS and Uppsala University fund [UPPMAX](http://uppmax.uu.se). — UU’s supercomputing centre. +- Compute and Storage is a unit of [NBIS](https://nbis.se/), the national bioinformatics support infrastructure and a SciLifeLab platform, and is hosted by UPPMAX. + +## UPPMAX missions +- Mission for NAISS: to provide a quality high-performance computing environment nationally + - General purpose: Rackham + - Data intensive. Rackham/Bianca + - Sensitive data (human sequences): Bianca +- Mission for UU: to provide relevant services to Uppsala University + - For UU researchers and education: Snowy cluster +- Mission for NBIS/SciLifeLab: to provide the best possible environment for data-driven life science, especially bioinformatics + +## UPPMAX support staff + +- System experts + - Hardware and system +- Application experts + - software and usage + - compute and storage project allocations + - user accounts + +## The Application experts not dealing with allocation and accounts + +- Transfer HPC competence + - intro courses like this +- Transfer computational know-how within the respective fields + - Physics, chemistry, bioinformatics, geoscience, coding + - [About the application-experts](https://www.uppmax.uu.se/about-us/application-experts/) +- In other words: + - installing software, + - troubleshooting, + - general advice, + - possibly make jobs more efficient. +- Email support for any query: [support@uppmax.uu.se](mailto:support@uppmax.uu.se) or use [support form](https://supr.naiss.se/support/) + - “Tickets” are distributed according to competence among the staff + diff --git a/_sources/rackham.md.txt b/_sources/rackham.md.txt new file mode 100644 index 0000000..5f232e5 --- /dev/null +++ b/_sources/rackham.md.txt @@ -0,0 +1,3 @@ +#Rackham +Rackham is used + diff --git a/_sources/slurm_intro.md.txt b/_sources/slurm_intro.md.txt new file mode 100644 index 0000000..2b11031 --- /dev/null +++ b/_sources/slurm_intro.md.txt @@ -0,0 +1,491 @@ +# Using the compute nodes +```{objectives} +- This is a short introduction in how to reach the compute nodes +- Wednesday afternoon is wedded to this topic! +``` + +```{instructor-note} +- Approx timing: 13.30-14.30 (10 min break) +- Theory +- Hands-on +``` + +```{attention} +- For now, **this course**, we use the **material on this present page**. +- A SLURM introduction can otherwise be found here: +``` + +```{note} +- project number: ``naiss2024-22-49`` +``` + +## The compute nodes + +When you are logged in, you are on a login node. +There are two types of nodes: + +Type |Purpose +------------|-------------------------- +Login node |Start jobs for worker nodes, do easy things. +Compute nodes |Do hard calculations, either from scripts of an interactive session. + + + +```{mermaid} + + graph TB + + Node1 -- interactive --> SubGraph2Flow + Node1 -- sbatch --> SubGraph2Flow + subgraph "Snowy" + SubGraph2Flow(calculation nodes) + end + + thinlinc -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + terminal/thinlinc -- usr --> Node1 + terminal -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + Node1 -- usr-sensXXX + 2FA + no VPN ----> SubGraph1Flow + + subgraph "Bianca" + SubGraph1Flow(Bianca login) -- usr+passwd --> private(private cluster) + private -- interactive --> calcB(calculation nodes) + private -- sbatch --> calcB + end + + subgraph "Rackham" + Node1[Login] -- interactive --> Node2[calculation nodes] + Node1 -- sbatch --> Node2 + end +``` + +## Slurm, sbatch, the job queue +- Problem: _1000 users, 300 nodes, 5000 cores_ +- We need a **queue**: + + - [Slurm](https://slurm.schedmd.com/) is a job scheduler + +- You define **jobs** to be run on the compute nodes and therefore sent to the queue. + +### Jobs +- Job = what happens during booked time +- Described in + - a script file or + - the command-line (priority over script) +- The definitions of a job: + - Slurm parameters (**flags**) + - Load software modules + - (Navigate in file system) + - Run program(s) + - (Collect output) + - ... and more + + +```{admonition} "Some keywords" +- A program may run _serially_ and then needs only ONE _compute thread_, which will occupy 1 core, which is a physical unit of the CPU on the node. + - You should most often just book 1 core. If you require more than 7 GB you can allocate more cores and you will get multiples of 7 GB. +- A program may run in _parallel_ and then needs either several _threads_ or several _tasks_, both occupying several cores. + - If you need all 128 GB RAM (actually 112) or all 16 cores for your job, book a complete node. +``` + +### Slurm parameters +- 1 mandatory setting for jobs: + - Which compute project? (`-A`) +- 3 settings you really should set: + - Type of queue or partition? (`-p`) + - ``core`` for most jobs and **default**! + - ``node`` for larger jobs + - for short development jobs and tests: ``devcore``, ``devel``) + - How many cores? (`-n`) + - up to 16 for core job + - How long at most? (`-t`) +- If in doubt: + - `-p core` + - `-n 1` + - `-t 10-00:00:00` + +### The queue + +- How does the queue work? + +- Let's look graphically at jobs presently running. + +![Image](./img/queue1.png) + +- *x-axis: cores, one thread per core* +- *y-axis: time* +

+ +- We see some holes where we may fit jobs already! +- Let's see which type of jobs that can fit! + +![Image](./img/queue2.png) + +
+ +- 4 one-core jobs can run immediately (or a 4-core wide job). + + - *The jobs are too long to fit at core number 9-13.* + +![Image](./img/queue3.png) +
+ +- A five-core job has to wait. + + - *Too long to fit in cores 9-13 and too wide to fit in the last cores.* + +- Easiest to schedule *single-threaded*, short jobs + + +```{tip} + +- You don't see the queue graphically, however. +- But, overall: + - short and narrow jobs will start fast + - test and development jobs can get use of specific development nodes if they are shorter than 1 hour and uses up to two nodes. + - waste of resources unless you have a parallel program or need all the memory, e.g. 128 GB per node +``` + +### Core-hours + +- Remember that you are charged CPU-hours according to booked #cores x hours +- Example 1: 60 hours with 2 cores = 120 CPU-hours +- Example 2: 12 hours with a full node = 192 hours + - Waste of resources unless you have a parallel program using all cores or need all the memory, e.g. 128 GB per node + +### Choices +- Work interactively with your data or develop or test + - Run an **Interactive session** + - ``$ interactive ...`` +- If you _don't_ need any live interaction with your workflow/analysis/simulation + - Send your job to the slurm job batch (sbatch) + - `$ sbatch ` or + - `$ sbatch ` + +```{mermaid} +flowchart TD + UPPMAX(What to run on which node?) + operation_type{What type of operation/calculation?} + interaction_type{What type of interaction?} + login_node(Work on login node) + interactive_node(Work on interactive node) + calculation_node(Schedule for calculation node) + + UPPMAX-->operation_type + operation_type-->|light,short|login_node + operation_type-->|heavy,long|interaction_type + interaction_type-->|Direct|interactive_node + interaction_type-->|Indirect|calculation_node +``` + +### What kind of compute work are you doing? +- **Compute bound** + - you use mainly CPU power + - does the software support threads or MPI? + - **Threads/openMP** are rather often supported. **Use several cores!** + - **MPI** (Message Passing Interface) allows for inter-node jobs but are seldom supported for bioinformatics software. **You could use several nodes!** +- **Memory bound** + - if the bottlenecks are allocating memory, copying/duplicating + - use more cores up to 1 node, perhaps using a "fat" node. + +```{admonition} "Slurm Cheat Sheet" + +- ``-A`` project number +- ``-t`` wall time +- ``-n`` number of cores +- ``-N`` number of nodes (can only be used if your code is parallelized with MPI) +- ``-p`` partition + - ``core`` is default and works for jobs narrower than 16 cores + - ``node`` can be used if you need the whole node and its memory + +``` + +### Walltime at the different clusters + +- Rackham: 10 days +- Snowy: 30 days +- Bianca: 10 days + + +## Interactive jobs +- Most work is most effective as submitted jobs, but e.g. development needs responsiveness +- Interactive jobs are high-priority but limited in `-n` and `-t` +- Quickly give you a job and logs you in to the compute node +- Require same Slurm parameters as other jobs +- Log in to compute node + - `$ interactive ...` +- Logout with `-D` or `logout` + +- To use an interactive node, in a terminal, type: + +```bash +interactive -A [project name] -p core -n [number_of_cores] -t [session_duration] +``` + +For example: + +```bash +interactive -A naiss2024-22-49 -p core -n 2 -t 8:0:0 +``` + +This starts an interactive session using project `naiss2024-22-49` +that uses 2 cores and has a maximum duration of 8 hours. + + + +### Try interactive and run RStudio + +- We recommend using at least two cores for RStudio, and to get those resources, you must start an interactive job. + +```{type-along} + +- Use **ThinLinc** + +- Start **interactive session** on compute node (2 cores) +- If you already have an interactive session going on use that. + - If you don't find it, do + + ``$ squeue`` + + - find your session, ssh to it, like: + + ``$ ssh r483`` + +- If you have no ongoing session: + +``$ interactive -A naiss2024-22-49 -p devcore -n 2 -t 60:00`` + +- Once the interactive job has begun you need to load needed modules, even if you had loaded them before in the login node +- You can check which node you are on? + + `$ hostname` + +- Also try: + + `$ srun hostname` + + - This will give several output lines resembling the number of cores you allocated. + - How many in this case?? + + + ``[bjornc@r483 ~]$`` + +- Load an RStudio module and an R_packages module (if not loading R you will have to stick with R/3.6.0) and run "rstudio" from there. + + `$ ml R_packages/4.2.1` + + `$ ml RStudio/2022.07.1-554` + + +- **Start rstudio**, keeping terminal active (`&`) + + `$ rstudio &` + +- Slow to start? +- Depends on: + - number of packages + - if you save a lot of data in your RStudio workspace, to be read during start up. + +- **Quit RStudio**! +- **Log out** from interactive session with `-D` or `logout` or `exit` +``` + +## Job scripts (batch) + +- Batch scripts can be written in any scripting language. We will use BASH +- Make first line be `#!/bin/bash` in the top line + - It is good practice to end the line with ``-l`` to reload a fresh environment with no modules loaded. + - This makes you sure that you don't enable other software or versions that may interfere with what you want to do in the job. +- Before the job content, add the batch flags starting the lines with the keyword `#SBATCH`, like: + - ``#SBATCH -t 2:00:00`` + - ``#SBATCH -p core`` + - ``#SBATCH -n 3`` +- `#` will be ignored by `bash` and can run as an ordinary bash script +- if running the script with the command `sbatch + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

NAISS SENS and Bianca

+
+

Objectives

+
    +

  • We’ll briefly get an overview of kinds of sensitive data

    • +

  • … and the Bianca system

    • +
+
+
+

The Bianca workshop

+ +
+
+

Instructor note

+
    +

  • Approx timing: 14.45-15.30 (15 min Coffee break)

    +
      +

    • Can perhaps be squeezed to 15 minutes after the coffee break

      • +
    +
    • +

  • Information

    • +
+
+
+

See also

+
    +

  • Bianca portal: http://docs.uppmax.uu.se/cluster_guides/bianca_portal/

    • +
+
+
+

Sensitive personal data

+ +
+
+

Apply for project

+

Open NAISS SENS Rounds

+
+
+

Bianca

+
    +

  • Bianca is a great platform for computationally intensive research on sensitive personal data. It can also be useful for:

    +
      +

    • national and international collaboration on sensitive personal data (without a high compute need)

      • +

    • other types of sensitive data

      • +
    +
    • +

  • Bianca is not good for:

    +
      +

    • storing data

      • +

    • publishing data

      +
        +

      • unless the dataset is very popular among Bianca users, e.g. Swegen, SIMPLER

        • +
      +
      • +
    +
    • +
+
+
+

Bianca’s design

+
    +

  • Bianca was designed to:

    +
      +

    • make accidental data leaks difficult

      • +

    • make correct data management as easy as possible

      • +

    • emulate the HPC cluster environment that SNIC/NAISS users were familiar with

      • +

    • provide a maximum amount of resources

      • +

    • satisfy regulations.

      • +
    +
    • +
+
+

Bianca has no Internet

+

… but we have “solutions”

+

Image

+
    +

  • Bianca is only accessible from within Sunet (i.e. from university networks).

    • +

  • Use VPN outside Sunet. Link to VPN for UU

    +
      +

    • You can get VPN credentials from all Swedish universities.

      • +
    +
    • +
+
+
    +

  • The whole Bianca cluster (blue) contains hundreds of virtual project clusters (green), each of which is isolated from each other and the Internet.

    • +

  • Data can be transferred to or from a virtual project cluster through the Wharf, which is a special file area that is visible from the Internet.

    • +
+
+

See also

+ +
+
+
+
+

Introduction course

+ +
+

Keypoints

+
    +

  • If you handle sensitive data, apply for a NAISS-SENS project

    • +

  • SENS projects will get accounts on Bianca

    • +

  • Bianca has no internet itself but there are solutions like:

    +
      +

    • wharf

      • +

    • transit server

      • +

    • many installed software

      • +
    +
    • +

  • Ask support if you need additional software tools

    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/clusters.html b/clusters.html new file mode 100644 index 0000000..b02aa7d --- /dev/null +++ b/clusters.html @@ -0,0 +1,183 @@ + + + + + + + Clusters — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Clusters

+

We have a number of compute clusters:

+
Rackham, reserved for SNIC projects
+Snowy, reserved for SNIC projects
+Miarka, reserved for Scilifelab production
+Bianca, a part of SNIC-SENS
+UPPMAX Cloud, a part of SNIC Science Cloud
+
+
+

The storage systems we have provide a total volume of about 20 PB, the equivalent of nearly 15 billion 3.5-inch floppy disks or 40,000 years of 128-bit encoded music. Read more on the storage systems page.

+
+

Rackham

+

Rackham

+
+
+

Snowy

+

Snowy

+
+
+

Bianca

+

Bianca

+
+
+

Miarka

+

Miarka

+
+
+

The UPPMAX Cloud

+

UPPMAX cloud

+
+
+

System usage

+

System usage

+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/courses.html b/courses.html new file mode 100644 index 0000000..8f54e8e --- /dev/null +++ b/courses.html @@ -0,0 +1,302 @@ + + + + + + + Courses and workshops — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Courses and workshops

+
+

Instructor note

+
    +

  • Approx timing: 14.40-14:50

    • +

  • Theory

    • +
+
+
+

Introductory Linux courses

+
+

Intro to UPPMAX (This one)

+ +
+
+

Awk workshop

+ +
+
+
+

Other UPPMAX courses

+
+

Intro to Bianca

+ +
+
+

Basic Singularity

+ +
+
+

Introduction to running R, Python, and Julia in HPC

+ +
+
+

Git course

+ +
+
+

Python at UPPMAX and HPC2N

+ +
+
+

Programming formalisms

+ +
+
+
+

Courses at other NAISS/SNIC Centers

+ +
+
+

CodeRefinery

+ +
+
+

ENCCS (EuroCC National Competence Centre Sweden

+ +
+ +
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/exercises.html b/exercises.html new file mode 100644 index 0000000..34a3cc8 --- /dev/null +++ b/exercises.html @@ -0,0 +1,150 @@ + + + + + + + Exercises — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Exercises

+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/genindex.html b/genindex.html new file mode 100644 index 0000000..6f50812 --- /dev/null +++ b/genindex.html @@ -0,0 +1,154 @@ + + + + + + Index — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+
    +
  • »
    • +
  • Index
    • +
  • +
    • +
+
+
+
+
+ + +

Index

+ +
+ +
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/hpc.html b/hpc.html new file mode 100644 index 0000000..d4a5f2d --- /dev/null +++ b/hpc.html @@ -0,0 +1,344 @@ + + + + + + + High Performance Computing — HPC — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

High Performance Computing — HPC

+
+

Objectives

+
    +

  • Let’s recap and go a little further into the UPPMAX hardware!

    • +
+
+
+

See also

+

Uppmax clusters

+
+
+

Instructor note

+
    +

  • Approx timing: 11.35-11:40

    • +

  • Information

    • +
+
+
+

HPC, HTC and MTC

+
    +

  • The Buzz word is HPC or High Performance Computing, but this is rather narrow focusing on fast calculation, i.e. processors and parallelism

    • +

  • Many of your projects are more focusing on high throughput, large memory demands and many tasks.

    • +

  • Here is a list of the three most common Computing paradigms:

    • +

  • HPC: High Performance Computing — Focus on floating point operations per second (FLOPS, flops or flop/s)

    +
      +

    • characterized as needing large amounts of computing power for short periods of time

      • +
    +
    • +

  • HTC: High-Throughput Computing —

    +
      +

    • operations or jobs per month or per year.

      • +

    • more interested in how many jobs can be completed over a long period of time instead of how fast.

      • +

    • independent, sequential jobs that can be individually scheduled o

      • +
    +
    • +

  • MTC: Many-task Computing — emphasis of using many computing resources over short periods of time to accomplish many computational tasks

    +
      +

    • bridge the gap between HTC and HPC.

      • +

    • reminiscent of HTC, but including both dependent and independent tasks), where the primary metrics are measured in seconds (e.g. FLOPS, tasks/s, MB/s I/O rates), as opposed to operations (e.g. jobs) per month.

      • +

    • high-performance computations comprising multiple distinct activities, coupled via file system operations.

      • +
    +
    • +
+
+
+

What is a cluster?

+
    +

  • A network of computers, each computer working as a node.

    • +

  • From small scale RaspberryPi cluster…

    • +
+

RaspBerry

+
    +

  • To supercomputers like Rackham.

    • +
+

Rackham

+
    +

  • Each node contains several processor cores and RAM and a local disk called scratch.

    • +
+

Node

+
    +

  • The user logs in to login nodes via Internet through ssh or Thinlinc.

    +
      +

    • Here the file management and lighter data analysis can be performed.

      • +
    +
    • +
+

RaspBerry

+

RaspBerry

+
    +

  • The calculation nodes has to be used for intense computing.

    +
      +

    • “Normal” softwares use one core.

      • +

    • Parallelized software can utilize several cores or even several nodes. Keywords signalizing this are e.g.:

      +
        +

      • “multi-threaded”, “MPI”, “distributed memory”, “openMP”, “shared memory”.

        • +
      +
      • +

    • To let your software run on the calculation nodes

      +
        +

      • start an “interactive session” or

        • +

      • “submit a batch job”.

        • +

      • More about this in today’s introduction to jobs.

        • +
      +
      • +
    +
    • +
+
+
+

Storage basics

+
    +

  • All nodes can access:

    +
      +

    • Your home directory on Domus or Castor

      • +

    • Your project directories on Crex or Castor

      • +

    • Its own local scratch disk (2-3 TB)

      • +
    +
    • +

  • If you’re reading/writing a file once, use a directory on Crex or Castor

    • +

  • If you’re reading/writing a file many times…

    +
      +

    • Copy the file to ”scratch”, the node local disk:

      • +
    +
    cp myFile $SNIC_TMP
+
    +
    +
    • +
+
+
+

The UPPMAX hardware

+
+

Clusters

+
    +

  • We have a number of compute clusters:

    +
      +

    • Rackham +, reserved for SNIC projects

      • +

    • Snowy, GPU, long jobs reserved for UPPMAX projects and Education

      • +

    • Bianca +, a part of SNIC-SENS

      • +

    • Miarka, reserved for Scilifelab production

      • +

    • UPPMAX cloud, a part of SNIC Science Cloud

      • +
    +
    • +

  • User guides

    • +

  • The storage systems we have provide a total volume of about 25 PB, the equivalent of 50,000 years of 128-bit encoded music. Read more on the storage systems page.

    • +
+
+
+

UPPMAX storage system names (projects & home directories)

+
    +

  • Rackham storage: Crex & Domus

    • +

  • Bianca storage: Castor & Cygnus

    • +

  • NGI production system (Miarka): Vulpes

    • +

  • NGI delivery server: Grus

    • +

  • Off-load storage: Lutra

    • +
+
+
+

System usage

+

System usage

+ +
+
+

A little bit more about Snowy

+
    +

  • User guide.

    +
      +

    • There is a local compute round for UU users applying for Snowy in SUPR.

      • +

    • GU (courses) applications (including GU GPU usage) are not done in SUPR, but are supposed to be routed through the service desk.

      • +

    • The details can be found at the Getting started page.

      • +
    +
    • +
+
+
+

About Bianca?

+
    +

  • Wait for it!

    • +
+
+

Keypoints

+
    +

  • UPPMAX has several clusters

    +
      +

    • each having its focus and limitation or possibilites

      • +

    • access is determined by type of project

      • +
    +
    • +
+
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 0000000..1f3485f --- /dev/null +++ b/index.html @@ -0,0 +1,286 @@ + + + + + + + Introduction to UPPMAX — first day lessons — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Introduction to UPPMAX — first day lessons

+

Here you will find some of the documentation for the UPPMAX resources in an easy to find place.

+

We will present and refer a lot to our new documentation at http://docs.uppmax.uu.se/

+
+

Intro

+ +
+
+

Overview

+ +
+
+

Get started

+ +
+
+

More about UPPMAX

+ +
+
+

Using UPPMAX

+ +
+
+

NAISS SENS

+ +
+
+

Courses and workshops

+ +
+
+

Summary

+ +
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/intro.html b/intro.html new file mode 100644 index 0000000..c1d07eb --- /dev/null +++ b/intro.html @@ -0,0 +1,313 @@ + + + + + + + Welcome on-site! — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Welcome on-site!

+
+

Course punchline

+
    +

  • Move from being a Linux novice to an UPPMAX expert.

    • +
+
+
+

Learning outcomes

+
    +

  • Get overview of UPPMAX and NAISS

    • +

  • Work with your files in a LINUX environment

    • +

  • Learn to use the UPPMAX resources in a good way

    • +

  • Learn the basics in the Python programming language

    • +
+
+
+

Lesson material

+ +
+
+

Questions

+
    +

  • Questions very much related to the present talk during the lessons

    +
      +

    • best posed directly by voice

      • +
    +
    • +

  • Questions requiring more effort to explain or answer

    + +
    • +
+
+
+

Type along

+
    +

  • You will be able to follow along using the built-in terminals in all OS:s.

    +
      +

    • Linux/Mac Terminal

      • +

    • Windows: Powershell or command prompt

      • +
    +
    • +

  • For your day-to-day work you may benefit from other applications, see the Login session

    • +
+
+
+

Exercises

+
    +

  • Exercises may be solved individually or in small groups in classroom

    • +
+
+
+

Example of arrangement on your screen

+
    +

  • HackMD

    • +

  • lesson material

    • +

  • your own terminal

    • +
+
+
+

Lesson Modules

+
+

Linux I at 9-12 Monday

+

Links in the left menu

+
    +

  • Overview

    • +

  • Logging in to UPPMAX

    • +

  • Linux basics

    • +

  • Using the command line

    • +
+
+
+

Intro to UPPMAX 13-16 Monday

+

Links in the left menu

+
    +

  • Organisation

    • +

  • HPC computing

    • +

  • NAISS

    • +

  • Using software and tools

    • +

  • Submitting jobs to calculation nodes (intro)

    • +

  • Working on UPPMAX

    • +

  • Courses and workshops

    • +

  • Short intro to Bianca and NAISS-SENS for the interested

    • +
+
+
+

Linux II 9-12 Tuesday

+

Linux II pdf

+
    +

  • Using the command line level 2

    • +

  • Input, output and pipes

    • +

  • Wildcards

    • +

  • The find command

    • +

  • Links

    • +
+
+
+

Linux III 13-16 Tuesday

+

Linux III pdf

+
    +

  • The grep command

    • +

  • Using the command line level 3

    • +

  • Variables, parameters and $(…)

    • +

  • Filenames

    • +
+
+
+

Bash Scripts 9-12 Wednesday

+

BASH scripts pdf

+
    +

  • Bash scripting, why’s and basics

    • +

  • Parameters

    • +

  • The IF statement

    • +

  • File tests

    • +

  • FOR and WHILE loops

    • +

  • Background processes and job control

    • +
+
+
+

Slurm at UPPMAX 13-16 Wednesday

+

Slurm pdf

+
    +

  • The workload manager

    • +

  • Quick testing and debugging as interactive jobs

    • +

  • Monitoring and modifying jobs

    • +

  • Priority

    • +

  • Efficiency

    • +

  • Script and workflow examples

    • +

  • Parallel computing and GPU

    • +

  • Snakemake and nextflow

    • +
+
+
+

Python I 9-12 Thursday

+

Python material

+
    +

  • Introduction

    • +

  • Getting started

    • +

  • Data types

    • +

  • Control flow statements

    • +
+
+
+

Python II 13-16 Thursday

+

Python material

+
    +

  • Functions

    • +

  • Modules

    • +

  • Command line arguments and IO

    • +

  • Object-oriented Python (bonus)

    • +
+
+
+
+

Icebreaker in HackMD

+ +
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/intro_online.html b/intro_online.html new file mode 100644 index 0000000..94bd510 --- /dev/null +++ b/intro_online.html @@ -0,0 +1,407 @@ + + + + + + + Welcome on-line! — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Welcome on-line!

+
+

Course punchline

+
    +

  • Move from being a Linux novice to an UPPMAX expert.

    • +
+
+
+

Course aim

+
    +

  • UPPMAX application experts want to share their skills in a 4-day series of lectures.

    • +

  • If you already have the fundamentals down, you are still sure to enjoy the tips and tricks in the later parts of the course.

    • +

  • Hence we’ve made it possible to sign up only for the days that are interesting to you.

    • +
+
+
+

Learning outcomes

+
    +

  • Get overview of UPPMAX and NAISS

    • +

  • Work with your files in a LINUX environment

    • +

  • Learn to use the UPPMAX resources in a good way

    • +

  • Learn the basics in the Python programming language

    • +
+
+
+

Lesson material

+ +
+
+

New documentation page

+ +
+
+

Questions

+ +
+

Questions requiring more effort to explain or answer

+ +
+
+
+

Exercises

+
    +

  • Exercises may be solved individually or in small groups in breakout rooms.

    • +
+
+
+

Type along

+
    +

  • You will be able to follow along using the built-in terminals in all OSs.

    +
      +

    • Linux/Mac Terminal

      • +

    • Windows: Powershell or command prompt

      • +
    +
    • +

  • For your day-to-day work you may benefit from other applications, see the Login session

    • +
+
+
+

The topics

+
+

Linux I at 9-12 Monday

+
    +

  • Overview

    • +

  • Logging in to UPPMAX

    • +

  • Linux basics

    • +

  • Using the command line

    • +
+
+
+

Intro to UPPMAX 13-16 Monday

+
    +

  • Organisation

    • +

  • HPC computing

    • +

  • SNIC

    • +

  • Using software and tools

    • +

  • Submitting jobs to calculation nodes (intro)

    • +

  • Bianca cluster

    • +
+
+
+

Linux II 9-12 Tuesday

+
    +

  • Using the command line level 2

    • +

  • Input, output and pipes

    • +

  • Wildcards

    • +

  • The find command

    • +

  • Links

    • +
+
+
+

Linux III 13-16 Tuesday

+
    +

  • The grep command

    • +

  • Using the command line level 3

    • +

  • Variables, parameters and $(…)

    • +

  • Filenames

    • +
+
+
+

Bash Scripts 9-12 Wednesday

+
    +

  • Bash scripting, why’s and basics

    • +

  • Parameters

    • +

  • The IF statement

    • +

  • File tests

    • +

  • FOR and WHILE loops

    • +

  • Background processes and job control

    • +
+
+
+

Slurm at UPPMAX 13-16 Wednesday

+
    +

  • A workload manager

    • +

  • Queuing

    • +

  • Monitoring

    • +

  • in queue

    • +

  • while running

    • +

  • when finished

    • +

  • Job testing within the interactive or “devel” mode

    • +

  • Scripting

    • +
+
+
+

Python I 9-12 Thursday

+
    +

  • Introduction

    • +

  • Basics

    • +

  • Data types

    • +

  • Control structures

    • +

  • Writing and executing a program

    • +
+
+
+

Python II 13-16 Thursday

+
    +

  • Creating a program

    • +

  • Functions

    • +
+
+
+
+

Today

+
+

Preliminary schedule

+
    +

  • First time in this manner so timings are preliminary.

    • +
+

Schedule

+
    +

  • 9.00 Intro

    • +

  • 9.15 Overview

    • +

  • 9.30 Log in

    • +

  • 9.55 15 min coffee break

    • +

  • 10.10 Linux intro

    • +

  • 10.40 Linux basic commands

    • +

  • (c. 11.00 10 min break)

    • +

  • 11.30 Overview - part 2

    • +

  • 11.35 HPC

    • +

  • 11.40 NAISS and projects

    • +

  • 11.50 Q/A if time allows

    • +

  • 12.00 LUNCH

    • +

  • 13.00 Software and tools

    • +

  • 13.30 Using compute nodes

    • +

  • (c. 14.00 10 min break)

    • +

  • 14.30 Working on UPPMAX

    +
      +

    • Room for sharing tips

      • +

    • Then run to coffee break

      • +
    +
    • +

  • (14.45 NAISS-SENS and Bianca (Do we have any users here?))

    • +

  • 14.55 15 min coffee break

    • +

  • 15.10 NAISS-SENS and Bianca p2

    • +

  • 15.30 Courses and workshops

    • +

  • 15.35 Summary

    • +

  • 15.40 Q/A

    • +
+
+
+

The New documentation

+
    +

  • New look later in spring for the official UPPMAX webpage, now at <www.uppmax.uu.se>

    • +

  • We like the documentation/tutorials in another style: http://docs.uppmax.uu.se

    +
      +

    • boxes, graphs, programming code style

      • +
    +
    • +

  • Let’s look at the documentation!

    +
      +

    • Getting started

      • +

    • Cluster user guide

      • +

    • Software

      • +

    • Databases

      • +
    +
    • +
+
+
+

Style of todays lectures

+
    +

  • Information sessions

    • +

  • Case sessions with usage of the new documentation

    • +
+
+
+
+

Icebreaker in HackMD

+ +
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/linux.html b/linux.html new file mode 100644 index 0000000..2c0e38d --- /dev/null +++ b/linux.html @@ -0,0 +1,469 @@ + + + + + + + Linux — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Linux

+
    +

  • The “operating system” of the UPPMAX and most of the other clusters is Linux.

    • +
+
+

Questions

+
    +

  • What is Linux?

    • +

  • How to use the command line?

    • +
+
+
+

Objectives

+
    +

  • We’ll briefly get an overview of Linux

    +
      +

    • How the command line works

      • +

    • Some text editors

      • +

    • Things to be aware of

      • +
    +
    • +
+
+
+

What is Linux?

+

Content

+
    +

  • Daily speaking: The Linux Operating system is a UNIX like and UNIX compatible Operating system.

    • +

  • Linux is a “Kernel” on which many different programs can run.

    • +

  • The shell (bash, sh, ksh, csh, tcsh and many more) is one such program.

    • +
+

Content

+
    +

  • Actually, for it to be an OS, it is supplied with GNU software and other additions giving us the name GNU/Linux.

    +
      +

    • [Linux naming controversy]((https://en.wikipedia.org/wiki/GNU/Linux_naming_controversy)

      • +
    +
    • +
+

Content

+
    +

  • Linux has a multiuser platform at its base which means permissions and security comes easy.

    • +
+
+

Linux comes in different distributions, dialects or, say, flavours.

+
    +

  • UPPMAX runs CentOS and RedHat

    • +
+

Content

+
+
+
+

Using the command line

+
+

Command line with bash (Bourne Again Shell)

+
    +

  • A Unix shell and command language.

    • +

  • Often default shell

    • +
+

Content

+
    +

  • The command-line interface: the bash prompt $

    • +

  • bash can be seen as a program that finds and runs other programs

    • +

  • bash is scripting language that is referred to as a shell

    +
      +

    • (because it sits around the kernel making it easy to interact with)

      • +
    +
    • +
+

Content

+
+
+

The prompt

+

[info]$ program word1 word2 word3 […]

+
    +

  • [info] is configurable, and usually tells you who you are, on what system, and where in the file system.

    + +
    • +
+

folders

+
+
+

Example bash command

+
+mv inbox +



+program flags +



















+
+
    +

  • Terminal screen shows

    • +
+

+

Content

+
+
+

Tab Completion

+
+tab +






+
+
    +

  • Whenever you’re writing a path or filename on the bash prompt, you can strike the ‘tab’ key to ask Bash to complete what you’re writing.

    • +

  • Get in the habit of this — it will save you many hours!

    • +
+
+
+
+

Editing files with file/text editors

+

Content

+
+

gedit

+
    +

  • graphical user interface — GUI, needs X-server

    • +

  • Also graphical editor within MobaXterm

    • +
+
+
+

nano

+ + +
+
+
+

The graphical editors

+
    +

  • When starting the graphical versions of an editor, add & to be able to use the command line while program is open.

    +
      +

    • Ex: gedit &

      • +

    • If not, you can <Ctrl>+z and type bg to send program to background.

      • +
    +
    • +
+
+

Try gedit

+
    +

  • start gedit with gedit &

    • +

  • write something

    • +

  • save

    • +
+
+
- ``ctrl-C`` interupts a program or a command that is "stuck"
+- ``ctrl-Z`` pauses a program, can be continues in background (``bg``) or  foreground (``fg``)
+- ``ctrl-D`` quits some programs
+
+
+
+
+

Typical sources of error

+

Content

+
+

Warning

+
    +

  • Capitalization matters in file names and program names

    • +

  • Spaces matter.

    +
      +

    • Always have a space after the program name.

      • +

    • Don’t add spaces within file names.

      • +
    +
    • +

  • Check that you are in the right place in the file system.

    • +

  • File permissions. Check that the right read, write and execute permission are set. See next session.

    • +
+
+
+
+

Caution!!

+

Content

+
+

Warning

+
    +

  • There is no undo for:

    +
      +

    • copy (cp),

      • +

    • move (mv), and

      • +

    • remove (rm).

      • +
    +
    • +

  • Beware of overwriting files and deleting the wrong ones.

    • +
+
+
+

Note

+
    +

  • Tip: make “rm” ask if you really want to erase:

    +
      +

    • Within a session: Type in the command prompt

      +
      alias rm='rm -i'
+
      +
      +
      • +

    • Override asking with

      +
      rm –f <>
+
      +
      +
      • +

    • Edit file .bashrc in home directory by adding the alias line for this to start everytime.

      • +
    +
    • +

  • This will also work for mv and cp!

    • +
+
+
+

Note

+
    +

  • If you do destroy your data, email UPPMAX support, we may be able to help.

    • +
+
+
+

Keypoints

+
    +

  • Linux Operating system is a UNIX-like and UNIX compatible Operating system.

    • +

  • Typical command: +$ program word1 word2 word3 […]

    • +

  • Example of file editors

    +
      +

    • terminal

      +
        +

      • nano

        • +

      • vim

        • +

      • emacs

        • +
      +
      • +

    • graphical:

      +
        +

      • gedit

        • +
      +
      • +
    +
    • +

  • Tips

    +
      +

    • use Tab completion

      • +

    • capitalization and spaces matters

      • +

    • no undo:s for copying, moving and removing

      +
        +

      • Solution: alias rm='rm -i'

        • +
      +
      • +
    +
    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/linux_basics.html b/linux_basics.html new file mode 100644 index 0000000..0506772 --- /dev/null +++ b/linux_basics.html @@ -0,0 +1,821 @@ + + + + + + + Basic toolkit — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Basic toolkit

+

Caption

+
+

Objectives

+
    +

  • Let’s dig into the most important BASH commands

    • +

  • We’ll do a type-along session

    • +
+
+
+

We will cover these commands

+ +
+

Read files and change file properties

+
    +

  1. cat  print content on screen

    2. +

  2. head  print first part

    3. +

  3. tail  print last part

    4. +

  4. less  browse content

    5. +

  5. tar  compress or extract file

    6. +

  6. chmod  change file permissions

    7. +

  7. man  info about a command

    8. +
+
+
+
+

File system Navigation

+
+

pwd — where are you now? “Print name of current/Working Directory”

+
$ pwd
+
+$ pwd -P 
+
+
+
    +

  • -P gives you the physical path,

    +
      +

    • ignores how you got there

      • +
    +
    • +
+
+
+

ls — list directory contents

+

Type ls to display the contents of the current directory.

+
$ ls -a
+
+
+

-a also shows hidden files and directories.

+
$ ls -l
+
+
+

-l gives you listed and detailed information in alpha-numerical order

+
+

Tip

+
    +

  • the short-hand command ll will make a list as above, but highlight different file types with different colors

    • +
+
+
$ ls -lt
+
+
+

-lt sorts things by time modified.

+
$ ls –lrt
+
+
+

-r gives reversed order, so in this case newest in last line.

+
$ man ls
+
+
+
    +

  • for complete information about a command.

    • +

  • TIP: -$ man <command> works for almost any command!

    +
      +

    • scroll with arrows and quit with q.

      • +
    +
    • +
+
+
+

cd — Change the shell working Directory

+
    +

  • To change directory, use cd <target>

    • +
+
$ cd /proj/introtouppmax
+
+$ pwd
+
+$ ls
+
+$ cd labs
+
+$ pwd
+
+
+
+

Experiment with cd

+
    +

  • Experiment with cd.

    • +

  • Try adding <spaces> or extra / in various places.

    • +

  • Use tab completion to avoid typos and typing ls a lot.

    • +

  • Figure out the use of the following:

    • +
+
$ cd -
+
+$ cd ..
+
+$ cd
+
+$ cd ~
+
+
+
+ +
+
+
+

Copy, Create, Move

+
+

mkdir — make directories

+
+

Warning

+
    +

  • Make sure you’re in your home directory by cd ~

    • +
+
+
    +

  • Create a new directory uppmax-intro

    • +
+
$ cd ~
+$ mkdir uppmax-intro
+
+
+
    +

  • Go in there:

    • +
+
 $ cd uppmax-intro/
+
+
+
+
+

cp — copy files and directories

+
    +

  • Copy files with: cp <source> <target>

    • +

  • Set target to . to keep name and to point at present directory.

    • +
+
$ cp /proj/introtouppmax/labs/linux_tutorial/ .
+
+
+
    +

  • Well, that didn’t work. What does the error say?

    • +

  • So… try

    • +
+
$ cp -r /proj/introtouppmax/labs/linux_tutorial/ .
+
+
+

-r is for recursive, meaning including files and subdirectories!

+
    +

  • Move to your just created linux_tutorial/

    • +
+
$ cd linux_tutorial
+
+
+
    +

  • Make a copy of the file “newfile” in the same directory:

    • +
+
$ cp newfile copyfile
+
+
+
+
+

scp —secure copy (remote file copy program)

+
    +

  • Linux/MacOS: To copy data to/from Rackham, you can use scp from the terminal on your local machine:

    • +

  • Upload from present directory on local machine to your home directory on cluster.

    +
      +

    • Example (not done by us today)

      • +
    +
    • +
+
[bob@macbook]$ scp myinput bob@rackham.uppmax.uu.se:~/copyofmyinput
+
+[bob@macbook]$ scp myinput bob@rackham.uppmax.uu.se:~/                      # (keeping filename)
+
+
+
    +

  • Download

    • +
+
[bob@macbook]$ scp bob@rackham.uppmax.uu.se:~/mydata copyofmydata
+
+[bob@macbook]$ scp bob@rackham.uppmax.uu.se:~/mydata .                      # (keeping file name)
+
+
+
+
+

mv — move/rename file

+
    +

  • Moving files works just like copying files:

    • +

  • mv <source> <target>

    • +

  • Move the copy you just made to another place:

    • +
+
$ mv copyfile ../
+
+
+
    +

  • Rename it.

    • +
+
$ mv ../copyfile ../renamedfile
+
+
+
+
+
+

Archiving

+

tar — archiving and compression

+
    +

  • We’re going to need more files. Let’s extract the tar.gz file (tared and gzipped file)

    • +
+
$ tar -vxzf files.tar.gz
+
+
+
    +

  • The flags mean: +- verbosely +- extract +- gzipped +- filename

    • +

  • Order of flags may matter!

    +
      +

    • f should be in the start or in the end!

      • +
    +
    • +

  • You should see a list of files being extracted

    • +
+
+

Tip

+
    +

  • To compress use the flag -cinstead of -x

    • +
+
$ tar -czfv <tar file> <path/to/directory/file(s)-or-directory>
+
+
+
+
+
+

Deleting

+
+

rm — delete files or directories

+
+

Note

+
    +

  • Tip: make “rm” ask if you really want to erase:

    +
      +

    • Within a session: Type in the command prompt

      +
      alias rm='rm -i'
+
      +
      +
      • +

    • Override asking with

      +
      rm –f <>
+
      +
      +
      • +

    • Edit file “.BASHrc” in /home directory by adding the alias line for this to start every time.

      • +
    +
    • +

  • This will also work for mv and cp!

    • +
+
+
    +

  • Deleting files works just like copying or moving them: +rm <target>

    • +

  • Try it out:

    • +
+
$ rm ../renamedfile
+
+$ rm this_is_empty
+
+
+
    +

  • hmmmm…

    • +
+
+
+

rmdir — delete an empty directory

+
    +

  • We need another command to delete directories

    • +
+
$ rmdir this_is_empty
+
+$ rmdir this_has_a_file
+
+
+
    +

  • Again??

    • +

  • Is there a way to use rm to delete directories?

    • +
+ +
+
+
+

Help

+
+

man — manual, look up the right flags

+
    +

  • Nobody can remember whether it’s -R or -r for recursive, or if -f lets you choose a file or forces an action.

    • +
+
$ man ls 
+
+
+
    +

  • shows you how to use ls and all its options

    • +

  • Type /<keyword> to search for a keyword, use n (forward) and ´N` (backward) to scan through hits.

    • +

  • Scroll with arrows.

    • +

  • Type q to quit.

    • +
+
+

Exercise

+
    +

  • Spend some time now to browse the man pages for the commands you’ve just learned!

    • +
+
+ +
+
+
+

Let’s get wild with Wildcards

+

Caption

+
$ ls many_files
+
+$ ls many_files/*.txt
+
+$ ls many_files/file_1*1.docx
+
+
+
    +

  • Want to clean out temporary files ending in .tmp in all the subdirectories?

    • +
+
+

Warning

+
    +

  • NB! It could be wise to do ls -a */*.tmp first to see what will be deleted…

    • +
+
+
 $ rm */*.tmp
+
+
+
+

Exercise

+
    +

  • Exercise: Create a new directory and move all .txt files in many_files to it.

    • +
+
+
+
+

Reading files

+
    +

  • In Linux, you can (if you wish) also display files without being able to change them

    • +
+
$ cd old_project
+
+$ ls
+
+
+
    +

  • Hmm, which of these files are useful?

    • +
+
+

cat - concatenate files and print on the standard output

+

Caption

+
    +

  • cat dumps the contents of files to the terminal as text

    • +
+
 $ cat the_best
+
+
+
    +

  • Yummy!

    • +
+
 $ cat a
+
+
+
    +

  • What’s this???

    • +

  • Concatenate files with this wizardry:

    • +
+
 $ cat a the_best > combinedfiles.txt
+
+
+
    +

  • File a is written first and the_best is appended

    • +
+
+
+

head — display the top (heading) of a file

+

Caption

+
 $ head a
+
+
+
    +

  • You can choose how many lines to display (default 10)

    • +
+
 $ head -n 4 a
+
+
+
+
+

tail — display the end of a file

+

Caption

+
    +

  • Tail is the same as head, but for the other end.

    • +
+
 $ tail -n 5 a
+
+
+
    +

  • Handy to look at log files or to figure out the structure of a text file.

    • +
+
+
+

less — read a whole file

+
    +

  • cat doesn’t really work for long files

    • +
+
 $ less a
+
+
+
    +

  • Search with /<keyword> and n/N

    • +

  • Hit q to quit.

    • +

  • scroll with arrows.

    • +

  • man uses less!

    +

    “less is more”

    +
    • +
+
+
+
+

File permissions

+

Caption

+
+

Example

+
$ ls -l
+
+  drwxrwxr-x 2 marcusl marcusl 4096 Sep 19 2012 external_hdd 
+  -rwxr-xr-x 1 marcusl marcusl 17198 Jul 16 14:12 files.tar.gz
+
+
+
    +

  • Leading symbol:

    +
      +

    • d directory

      • +

    • - regular file

      • +

    • l symbolic link (more on this tomorrow)

      • +

    • Others exist, but you can ignore them for now

      • +
    +
    $ ls -l
+
+drwxrwxr-x 2 marcusl marcusl 4096 Sep 19 2012 external_hdd
+
+-rwxr-xr-x 1 marcusl marcusl 17198 Jul 16 14:12 files.tar.gz
+
    +
    +
    • +

  • Three sets of “rwx” permissions

    +
      +

    • rwx: r ead, w rite, ex ecute

      • +

    • User: the user account that owns the file (usually the one that created it)

      • +

    • Group: the group that owns the file (usually the project group in /proj/xyz or the user’s group elsewhere)

      • +

    • Others: everyone else on the system (literally a thousand strangers)

      • +
    +
    • +

  • r – read

    +
      +

    • Files: Read the contents of the file

      • +

    • Directories: List the files in the directory

      • +
    +
    • +

  • w – write

    +
      +

    • Files: Modify the file

      • +

    • Directories: Add, rename, or delete files in the directory

      • +
    +
    • +

  • x – execute

    +
      +

    • Files: Run the file as a program

      • +

    • Directories: Traverse the directory (e.g. with “cd”)

      • +
    +
    • +
+ +
+
+
+

Changing permissions

+

chmod — change file mode bits

+

If you own, i.e. created, the file or directory, you can modify the content

+
+

Common issues

+
    +

  • Files with w can be modified and destroyed by accident. Protect your data!

    • +

  • If you want to share data or scripts with a person not in your project (e.g. support staff like me), you can!

    • +

  • If you want to keep non-members from even seeing which files you have, you can!

    • +
+
+
+

Syntax

+

chmod <mode> <files>

+
    +

  • <mode> is of the form: For whom, Modify, What permission(s)

    • +

  • For whom?

    +
      +

    • u: user/owner

      • +

    • g: group, often the members to a certain project

      • +

    • o: others

      • +

    • a: all

      • +

    • if not set changes are applied for user AND group

      • +
    +
    • +

  • Modify?

    +
      +

    • +: add permissions,

      • +

    • -: remove

      • +

    • =: set equal to

      +
        +

      • = usually causes unmentioned bits to be removed except that a directory’s unmentioned set user and group ID bits are not affected.

        • +
      +
      • +
    +
    • +

  • What permissions?

    +
      +

    • r, w, x, i.e. the actual permission

      • +
    +
    • +
+
+

Examples

+
    +

  • <mode> can be e.g.:

    +
      +

    • u+x : lets You (owner) run a script you just wrote

      • +

    • -w : no write permissions for owner+group

      • +

    • warning: if w was already set for others it will be kept!!

      • +

    • +rw : let user and group members read and edit this file, not others if not already set

      • +

    • =xw : let group members go into your directory and put files there, but not see which files are there, others are not affected

      • +

    • a=xw : set xw for everyone

      • +
    +
    • +

  • chmod takes flags as usual, e.g.

    +
      +

    • -R for recursive (i.e. all files and sub-directories therein)

      • +
    +
    • +
+ +
+

chmod — Hands-on

+
    +

  • In your locally created linux_tutorial directory, find important files and old saved data that you wouldn’t want to lose (imagine).

    +
      +

    • Directories: important_results/, old_project/

      • +

    • File: last_years_data

      • +
    +
    • +

  • Use chmod to remove write permission from those files and directories (use the -R flag (not -r) to also do the files in the directories).

    +
      +

    • Take a moment to play around with chmod and explore the effects of permissions on files and directories.

      • +
    +
    • +
+
+ +

More about BASH command line and scripts on Tuesday and Wednesday!

+
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/linux_cases.html b/linux_cases.html new file mode 100644 index 0000000..32b1f96 --- /dev/null +++ b/linux_cases.html @@ -0,0 +1,384 @@ + + + + + + + Linux and basic toolkit — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Linux and basic toolkit

+
    +

  • The “operating system” of the UPPMAX and most of the other clusters is Linux.

    • +
+
+

Questions

+
    +

  • What is Linux?

    • +

  • How to use the command line?

    • +
+
+
+

Objectives

+
    +

  • We’ll briefly get an overview of Linux

    +
      +

    • How the command line works

      • +

    • Some text editors

      • +

    • Things to be aware of

      • +
    +
    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 10:10-10:40

    • +

  • Theory and type-along

    • +
+
+
+

See the documentation

+ +
+
+

Tip

+
    +

  • TAB complete:

    +
      +

    • Whenever you’re writing a path or filename on the bash prompt, you can strike the ‘tab’ key to ask Bash to complete what you’re writing.

      • +

    • Get in the habit of this — it will save you many hours!

      • +
    +
    • +
+

These commands are useful in the command line when something is stuck or a program is limiting you to do further work.

+
    +

  • ctrl-C interupts a program or a command that is “stuck”

    • +

  • ctrl-Z pauses a program, can be continues in background (bg) or foreground (fg)

    • +

  • ctrl-D quits some programs

    • +
+
+
+

Warning

+
    +

  • There is no undo for:

    +
      +

    • copy (cp),

      • +

    • move (mv), and

      • +

    • remove (rm).

      • +
    +
    • +

  • Beware of overwriting files and deleting the wrong ones.

    • +
+
+
+

Note

+
    +

  • Tip: make “rm” ask if you really want to erase:

    +
      +

    • Within a session: Type in the command prompt

      +
      alias rm='rm -i'
+
      +
      +
      • +

    • Override asking with

      +
      rm –f <>
+
      +
      +
      • +

    • Edit file .bashrc in home directory by adding the alias line for this to start everytime.

      • +
    +
    • +

  • This will also work for mv and cp!

    • +
+
+
+

Note

+
    +

  • If you do destroy your data, email UPPMAX support, we may be able to help.

    • +
+
+
+

Keypoints

+
    +

  • Linux Operating system is a UNIX-like and UNIX compatible Operating system.

    • +

  • Typical command: +$ program word1 word2 word3 […]

    • +

  • Example of file editors

    +
      +

    • terminal

      +
        +

      • nano

        • +

      • vim

        • +

      • emacs

        • +
      +
      • +

    • graphical:

      +
        +

      • gedit

        • +
      +
      • +
    +
    • +

  • Tips

    +
      +

    • use Tab completion

      • +

    • capitalization and spaces matters

      • +

    • no undo:s for copying, moving and removing

      +
        +

      • Solution: alias rm='rm -i'

        • +
      +
      • +
    +
    • +
+
+
+

Basic toolkit

+
+

Objectives

+
    +

  • Let’s dig into the most important BASH commands

    • +

  • We’ll do a type-along session

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 10:40-11.30

    • +

  • Type-alongs

    • +
+
+
+
+

We will cover these commands

+ +
+

Read files and change file properties

+
    +

  1. cat  print content on screen

    2. +

  2. head  print first part

    3. +

  3. tail  print last part

    4. +

  4. less  browse content

    5. +

  5. tar  compress or extract file

    6. +

  6. chmod  change file permissions

    7. +

  7. man  info about a command

    8. +
+
+

See the documentation

+ +
+
+

chmod — Hands-on

+
    +

  • In your locally created linux_tutorial directory, find important files and old saved data that you wouldn’t want to lose (imagine).

    +
      +

    • Directories: important_results/, old_project/

      • +

    • File: last_years_data

      • +
    +
    • +

  • Use chmod to remove write permission from those files and directories (use the -R flag (not -r) to also do the files in the directories).

    +
      +

    • Take a moment to play around with chmod and explore the effects of permissions on files and directories.

      • +
    +
    • +
+
+ + +

More about BASH command line and scripts on Tuesday and Wednesday!

+
    +

  • https://www.uppmax.uu.se/support/courses-and-workshops/uppmax-introductory-course/

    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/login.html b/login.html new file mode 100644 index 0000000..1ec62f5 --- /dev/null +++ b/login.html @@ -0,0 +1,379 @@ + + + + + + + Login — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Login

+
+

Questions

+
    +

  • How to reach UPPMAX clusters?

    • +

  • Can one use graphics?

    • +

  • Where do I arrive when I log in?

    • +
+
+
+

Objectives

+
    +

  • We’ll relate our login session to the specific “area” in the cluster.

    • +

  • We’ll go through platform specific (Mac/Linux/Windows) ways to log in to UPPMAX.

    • +

  • Try yourself!

    • +
+
+
+

Note

+

If you lack a user account, visit the Getting started page

+
+
+

The login

+
    +

  • When logging in to UPPMAX from your local computer you will arrive to your home folder at the login node.

    • +

  • This means that only light analysis and and calculations should be made here.

    • +

  • You will see this in the prompt after “@” as the clustername and a low number. For instance:

    +
       [<user>@rackham3 linux_tutorial]$
+
    +
    +
    • +

  • You willl ater learn how to reach the calculation nodes. Then the prompt states the node number with a single letter, like “r” for Rackham. For instance:

    +
       [<user>@r484 linux_tutorial]
+
    +
    +
    • +
+
+
+

MAC and LINUX users

+
+

Terminal

+
    +

  • Download XQuartz or other X11 server for Mac OS

    + +
    • +

  • Start built-in Terminal.

    • +
+
$ ssh -Y <username>@rackham.uppmax.uu.se
+
+
+
    +

  • “< >” prompts you to set the keyword specific for you or your needs. In the example above, this is basically your username.

    • +
+

Terminal

+
+
+

Graphical file manager

+ +

Caption

+
+

Problems with installations?

+

The built-in terminal without X11 is sufficient first days of the course!

+
+
+
+
+

Windows users

+
+

Most straight-forward

+
    +

  • Install a ssh (secure shell) program with built-in X11 and sftp file manager

    +
      +

    • MobaXterm

      +
        +

      • sftp frame makes it easy to move, upload and download files.

        • +
      +
      • +

    • See section below.

      • +
    +
    • +

  • If problem with the above installation, the putty.exe terminal is sufficient first days of the course!

    + +
    • +
+

Caption

+ +
+
+

MobaXterm

+

Caption

+
    +

  • Start local terminal and a SSH session by:

    • +
+
$ ssh -Y <username>@rackham.uppmax.uu.se
+
+
+

Caption

+
    +

  • Or even better, create and save a SSH session, as shown in image below.

    +
      +

    • This allows you to use MobaXterm as a file manager and to use the built-in graphical texteditor.

      • +

    • You can rename the session in the Bookmark settings tab.

      • +
    +
    • +
+

Caption

+
+
+

Graphical file manager (Windows)

+ +
+

Problems with installations?

+

The built-in terminal without X11 is sufficient first days of the course!

+
+
+
+
+

X11-forwarding from the command line (generally)

+
    +

  • Graphics can be sent through the SSH connection you’re using to connect

    +
      +

    • Use primarily ssh -Y <...> or secondary ssh -X <...>

      • +
    +
    • +

  • The X servers that enables graphics are needed, as mentioned above!

    +
      +

    • When starting a graphical program, a new window will open, but your terminal will be “locked”.

      • +

    • Run using “&” at the end to run it as a background process e.g. “xeyes &” or “gedit &”

      • +
    +
    • +
+

Caption

+
    +

  • Alternatively, use <ctrl>-z to put e.g. gedit to sleep and type “bg” to make last process in background.

    • +
+ +
+
+

ThinLinc (all platforms!)

+ +

Caption

+ +
+

Try Thinlinc from web

+
    +

  • Try the web version now if you don’t already have the software installed!

    • +
+
+
+

Keypoints

+
    +

  • You reach UPPMAX clusters either using a terminal or Thinlinc

    • +

  • Graphics are included in Thinlinc and from terminal if you have enabled X11

    • +

  • When you log in from your local computer you will always arrive at a login node with limited resources.

    • +

  • You reach the calculations nodes from within the login node (See Submitting jobs section)

    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/login2.html b/login2.html new file mode 100644 index 0000000..8ec48bb --- /dev/null +++ b/login2.html @@ -0,0 +1,344 @@ + + + + + + + Login — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Login

+
+

Questions

+
    +

  • How to reach UPPMAX clusters?

    • +

  • Where do I arrive when I log in? Login or calculation node?

    • +

  • What clients should/could I use for my work at UPPMAX?

    • +

  • How can I enable graphics?

    • +
+
+
+

Objectives

+
    +

  • We’ll go through platform specific (Mac/Linux/Windows) ways to log in to UPPMAX

    • +

  • See different clients

    • +

  • Enable graphics

    • +

  • Try yourself!

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 9:30-9:55

    • +

  • Demos/type-alongs

    • +
+
+
+

Note

+

If you lack a user account, visit the Getting started page

+
+
+

General understanding

+
    +

  • When logging in to UPPMAX from your local computer you will arrive to your home folder at the login node.

    • +

  • This means that only light analysis and and calculations should be made here.

    • +

  • You will see this in the prompt after “@” as the clustername and a low number. For instance:

    +
       [<user>@rackham3 linux_tutorial]$
+
    +
    +
    • +

  • You will later learn how to reach the calculation nodes. Then the prompt states the node number with a single letter, like “r” for Rackham. For instance:

    +
       [<user>@r484 linux_tutorial]
+
    +
    +
    • +
+
+
+

The login

+ +
    +

  • We will use Rackham for the course

    • +
+
+

See the documentation

+ +
+
+
+

Exercises

+ + + +
    +

  • If you receive errors or warnings, instead try:

    • +
+
$ ssh <username>@rackham.uppmax.uu.se
+
+
+
    +

  • If you do have X11 installed:

    • +
+
$ xeyes &
+
+
+
+

Try Thinlinc from web

+ +
+ +
+

Keypoints

+
    +

  • When you log in from your local computer you will always arrive at a login node with limited resources.

    • +

  • You reach the calculations nodes from within the login node (See Submitting jobs section)

    • +

  • You reach UPPMAX clusters either using a terminal client or Thinlinc

    • +

  • Graphics are included in Thinlinc and from terminal if you have enabled X11.

    • +

  • Which client to use?

    • +

  • Graphics and easy to use

    +
      +

    • ThinLinc

      • +
    +
    • +

  • Best integrated systems

    +
      +

    • Visual Studio Code has several extensions (remote, SCP, programming IDE:s)

      • +

    • Windows: MobaXterm is somewhat easier to use.

      • +
    +
    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/naiss.html b/naiss.html new file mode 100644 index 0000000..d96a118 --- /dev/null +++ b/naiss.html @@ -0,0 +1,174 @@ + + + + + + + NAISS and projects — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

NAISS and projects

+
+

Objectives

+
    +

  • Let’s step up and have an overview of NAISS

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 11.40-11:50

    • +

  • Information

    • +
+
+
+

See the documentation

+ +
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/objects.inv b/objects.inv new file mode 100644 index 0000000..67a01a4 Binary files /dev/null and b/objects.inv differ diff --git a/overview.html b/overview.html new file mode 100644 index 0000000..5d02d42 --- /dev/null +++ b/overview.html @@ -0,0 +1,316 @@ + + + + + + + Overview — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Overview

+
+

Questions

+
    +

  • What is UPPMAX?

    • +

  • What is SNIC/NAISS?

    • +

  • What is a cluster?

    • +
+
+
+

Objectives

+
    +

  • We’ll get an overview of UPPMAX and SNIC/NAISS and how a computer cluster works

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 9:15-9:30

    • +

  • Theory

    • +
+
+

UPPMAX = UppMACS

+ +
+

NAISS

+
    +

  • National Academic Infrastructure for Supercomputing in Sweden

    • +

  • Mission: to provide a quality high-performance computing environment nationally

    • +

  • Starting 1 January 2023, the National Academic Infrastructure for Supercomputing in Sweden (NAISS) is the new organization for high-performance computing, storage, and date services for academic users in Sweden.

    +
      +

    • Replaces SNIC — the Swedish National Infrastructure for Computing

      • +

    • Linköping university is the host organization for NAISS, but NAISS acts independently with a national perspective and responsibility.

      • +
    +
    • +

  • From the users perspective, there will initially only be minimal differences between the SNIC and NAISS regimes.

    + +
    • +

  • Application rounds: https://supr.naiss.se/round/

    • +

  • NAISS and Uppsala University fund UPPMAX — UU’s supercomputing center.

    • +
+
+
+

UPPMAX missions

+
    +

  • Runs the clusters placed in Uppsala.

    • +

  • More details in the afternoon about Organisational orienteering!

    • +
+
+
+

UPPMAX systems

+
    +

  • Clusters

    +
      +

    • Rackham (general purpose)

      +
        +

      • Snowy (Long runs and GPU:s)

        • +
      +
      • +

    • Bianca (sensitive data)

      +
        +

      • Miarka (NGI production)

        +
          +

        • genomic data from the sequencing machines are pre-processed before they are delivered to the researcher.

          • +
        +
        • +
      +
      • +
    +
    • +

  • Storage

    +
      +

    • On-load, directly connected to the clusters

      • +

    • Off-load, for large data not needed for computation analysis anymore

      • +
    +
    • +

  • Cloud

    +
      +

    • Dis (region EAST-1)

      • +
    +
    • +
+
+
+

High Performance Computing — HPC

+
+

What is a cluster?

+
    +

  • A network of computers, each computer working as a node.

    • +

  • From small scale RaspberryPi cluster…

    • +
+

RaspBerry

+
    +

  • To supercomputers like Rackham.

    • +
+

Rackham

+
    +

  • Each node contains several processor cores and RAM and a local disk called scratch.

    • +
+

Node

+
    +

  • The user logs in to login nodes via Internet through ssh or ThinLinc.

    +
      +

    • Here the file management and lighter data analysis can be performed.

      • +
    +
    • +
+

RaspBerry

+

RaspBerry

+
    +

  • The calculation nodes have to be used for intense computing.

    • +
+
+
+
+

Overview of the UPPMAX systems

+graph TB + + Node1 -- interactive --> SubGraph2Flow + Node1 -- sbatch --> SubGraph2Flow + subgraph "Snowy" + SubGraph2Flow(calculation nodes) + end + + thinlinc -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + terminal -- usr --> Node1 + terminal -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + Node1 -- usr-sensXXX + 2FA + no VPN ----> SubGraph1Flow + + subgraph "Bianca" + SubGraph1Flow(Bianca login) -- usr+passwd --> private(private cluster) + private -- interactive --> calcB(calculation nodes) + private -- sbatch --> calcB + end + + subgraph "Rackham" + Node1[Login] -- interactive --> Node2[calculation nodes] + Node1 -- sbatch --> Node2 + end +
+

Keypoints

+
    +

  • NAISS makes available large-scale high-performance computing resources, storage capacity, and advanced user support, for Swedish research.

    • +

  • UPPMAX runs the local resources placed at Uppsala Universtiy

    • +

  • A cluster consists of several inter-connected computers that can work individually or together.

    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/overview2.html b/overview2.html new file mode 100644 index 0000000..c4fb6b5 --- /dev/null +++ b/overview2.html @@ -0,0 +1,246 @@ + + + + + + + Overview — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Overview

+
+

Objectives

+
    +

  • Let’s recap and go a little deeper into the organization of NAISS/UPPMAX

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 11.30-11:35

    • +

  • Theory

    • +
+
+
+

Organisational orienteering

+
    +

  • VR/SRC and a consortium of universities funds NAISS — the National Academic Infrastructure for Supercomputing in Sweden

    • +

  • NAISS and Uppsala University fund UPPMAX. — UU’s supercomputing centre.

    • +

  • Compute and Storage is a unit of NBIS, the national bioinformatics support infrastructure and a SciLifeLab platform, and is hosted by UPPMAX.

    • +
+
+
+

UPPMAX missions

+
    +

  • Mission for NAISS: to provide a quality high-performance computing environment nationally

    +
      +

    • General purpose: Rackham

      • +

    • Data intensive. Rackham/Bianca

      • +

    • Sensitive data (human sequences): Bianca

      • +
    +
    • +

  • Mission for UU: to provide relevant services to Uppsala University

    +
      +

    • For UU researchers and education: Snowy cluster

      • +
    +
    • +

  • Mission for NBIS/SciLifeLab: to provide the best possible environment for data-driven life science, especially bioinformatics

    • +
+
+
+

UPPMAX support staff

+
    +

  • System experts

    +
      +

    • Hardware and system

      • +
    +
    • +

  • Application experts

    +
      +

    • software and usage

      • +

    • compute and storage project allocations

      • +

    • user accounts

      • +
    +
    • +
+
+
+

The Application experts not dealing with allocation and accounts

+
    +

  • Transfer HPC competence

    +
      +

    • intro courses like this

      • +
    +
    • +

  • Transfer computational know-how within the respective fields

    + +
    • +

  • In other words:

    +
      +

    • installing software,

      • +

    • troubleshooting,

      • +

    • general advice,

      • +

    • possibly make jobs more efficient.

      • +
    +
    • +

  • Email support for any query: support@uppmax.uu.se or use support form

    +
      +

    • “Tickets” are distributed according to competence among the staff

      • +
    +
    • +
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/rackham.html b/rackham.html new file mode 100644 index 0000000..c067137 --- /dev/null +++ b/rackham.html @@ -0,0 +1,149 @@ + + + + + + + <no title> — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +

#Rackham +Rackham is used

+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/search.html b/search.html new file mode 100644 index 0000000..ea82930 --- /dev/null +++ b/search.html @@ -0,0 +1,169 @@ + + + + + + Search — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+
    +
  • »
    • +
  • Search
    • +
  • +
    • +
+
+
+
+
+ + + + +
+ +
+ +
+
+ +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/searchindex.js b/searchindex.js new file mode 100644 index 0000000..713987f --- /dev/null +++ b/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({"docnames": ["bianca", "clusters", "courses", "exercises", "hpc", "index", "intro", "intro_online", "linux", "linux_basics", "linux_cases", "login", "login2", "naiss", "overview", "overview2", "rackham", "slurm_intro", "snic", "software", "specific", "start", "summary", "userguide", "working"], "filenames": ["bianca.md", "clusters.md", "courses.md", "exercises.rst", "hpc.md", "index.rst", "intro.md", "intro_online.md", "linux.md", "linux_basics.md", "linux_cases.md", "login.md", "login2.md", "naiss.md", "overview.md", "overview2.md", "rackham.md", "slurm_intro.md", "snic.md", "software.md", "specific.md", "start.md", "summary.md", "userguide.md", "working.md"], "titles": ["NAISS SENS and Bianca", "Clusters", "Courses and workshops", "Exercises", "High Performance Computing \u2014 HPC", "Introduction to UPPMAX \u2014 first day lessons", "Welcome on-site!", "Welcome on-line!", "Linux", "Basic toolkit", "Linux and basic toolkit", "Login", "Login", "NAISS and projects", "Overview", "Overview", "<no title>", "Using the compute nodes", "SNIC", "Software and tools", "Specific software user guides", "Getting started", "Summary", "User guide", "Working on UPPMAX"], "terms": {"we": [0, 1, 4, 5, 7, 8, 11, 12, 14, 17, 19, 21, 22, 24], "ll": [0, 8, 9, 10, 11, 12, 14, 19], "briefli": [0, 8, 10, 19], "get": [0, 4, 6, 7, 8, 10, 11, 12, 14, 17, 19, 24], "an": [0, 2, 4, 5, 6, 7, 8, 10, 13, 14, 21, 22, 23], "overview": [0, 6, 7, 8, 10, 13, 19, 22], "kind": [0, 5], "system": [0, 5, 8, 10, 12, 15, 17, 19, 21], "materi": [0, 5, 17], "next": [0, 8], "time": [0, 2, 4, 7, 9, 10, 12, 13, 14, 15, 17, 19, 24], "given": 0, "later": [0, 7, 12], "thi": [0, 4, 6, 7, 8, 9, 10, 11, 12, 15, 17, 19, 23], "spring": [0, 2, 7], "approx": [0, 2, 4, 10, 12, 13, 14, 15, 17, 19, 24], "14": [0, 2, 7, 9, 17, 24], "45": [0, 7, 24], "15": [0, 1, 7, 14], "30": [0, 7, 10, 12, 14, 15, 17, 19, 24], "min": [0, 7, 17], "coffe": [0, 7], "break": [0, 7, 17, 22], "can": [0, 4, 6, 7, 8, 9, 10, 11, 12, 14, 17, 19, 22, 23, 24], "perhap": [0, 17], "squeez": 0, "minut": [0, 2, 17], "after": [0, 8, 11, 12, 17], "inform": [0, 4, 7, 9, 13, 21, 24], "portal": 0, "http": [0, 5, 6, 7, 8, 10, 11, 12, 13, 14, 17, 18, 19, 22], "doc": [0, 5, 7, 10, 12, 13, 17, 22], "uppmax": [0, 8, 9, 10, 11, 12, 13, 17, 18, 19, 22, 23], "uu": [0, 4, 5, 7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 22], "se": [0, 5, 7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 22], "cluster_guid": [0, 17], "bianca_port": 0, "www": [0, 7, 8, 10, 11, 14, 18, 19], "snic": [0, 1, 4, 5, 7, 14, 21], "alloc": [0, 5, 17, 18, 21], "trace": 0, "now": [0, 7, 10, 11, 12, 17, 24], "live": [0, 17, 18], "e": [0, 4, 9, 11, 17, 18, 23], "g": [0, 4, 9, 10, 11, 17, 18, 23], "human": [0, 15], "genom": [0, 14, 23], "imag": [0, 11], "video": 0, "contain": [0, 2, 4, 14, 23], "health": 0, "registri": 0, "record": 0, "from": [0, 4, 6, 7, 9, 10, 14, 17, 19, 21, 23, 24], "healthcar": 0, "provid": [0, 1, 4, 14, 15, 18, 21], "more": [0, 1, 2, 6, 8, 9, 10, 14, 15, 17, 18, 24], "about": [0, 1, 9, 10, 14, 15, 17], "gdpr": 0, "protect": [0, 9, 24], "skydd": 0, "av": 0, "personuppgift": 0, "when": [0, 7, 8, 10, 11, 12, 17, 24], "doubt": [0, 17], "contact": 0, "your": [0, 4, 7, 8, 9, 10, 12, 17, 19, 21, 23, 24], "univers": [0, 14, 15, 18, 21], "offic": 0, "gener": [0, 5, 14, 15, 18, 22, 24], "must": [0, 17, 23], "process": [0, 6, 7, 11, 14], "agreement": 0, "between": [0, 4, 11, 14], "control": [0, 6, 7, 18], "open": [0, 8, 11, 12], "round": [0, 4, 14], "i": [0, 5, 9, 10, 11, 12, 15, 16, 18, 19, 21, 24], "great": 0, "platform": [0, 8, 12, 15], "computation": 0, "intens": [0, 4, 14, 15, 23, 24], "research": [0, 14, 15, 18, 22], "It": [0, 9, 17], "also": [0, 2, 8, 9, 10, 11, 17, 18, 19, 21], "us": [0, 2, 4, 6, 7, 9, 10, 12, 14, 15, 16, 18, 21, 22, 24], "nation": [0, 5, 14, 15, 18], "intern": [0, 18], "collabor": 0, "without": [0, 9, 11, 17, 24], "high": [0, 5, 15, 17, 22, 24], "comput": [0, 1, 5, 6, 7, 11, 12, 15, 18, 21, 22, 23, 24], "need": [0, 4, 8, 9, 11, 14, 17, 18, 19, 21, 22, 24], "other": [0, 5, 6, 7, 8, 9, 10, 11, 15, 18, 24], "type": [0, 4, 5, 8, 9, 10, 11, 12, 19, 24], "good": [0, 6, 7, 17, 19, 24], "store": 0, "publish": 0, "unless": [0, 17, 19, 24], "dataset": 0, "veri": [0, 6, 8, 17, 19, 24], "popular": 0, "among": [0, 9, 10, 15], "user": [0, 4, 7, 8, 9, 10, 12, 14, 15, 17, 19, 22, 24], "swegen": [0, 23], "simpler": 0, "wa": [0, 9, 18], "make": [0, 8, 10, 11, 14, 15, 17, 22, 24], "accident": 0, "leak": 0, "difficult": 0, "correct": 0, "manag": [0, 4, 6, 7, 14, 19, 23], "easi": [0, 5, 8, 11, 12, 17], "possibl": [0, 7, 15], "emul": 0, "hpc": [0, 5, 6, 7, 13, 15], "cluster": [0, 5, 7, 8, 9, 10, 11, 12, 13, 15, 19, 21, 22], "environ": [0, 2, 6, 7, 11, 12, 14, 15, 17, 18, 24], "were": [0, 11], "familiar": 0, "maximum": [0, 17], "amount": [0, 4], "resourc": [0, 4, 5, 6, 7, 11, 12, 14, 17, 18, 19, 21, 22, 24], "satisfi": 0, "regul": 0, "have": [0, 1, 4, 7, 8, 9, 11, 12, 13, 14, 17, 19, 24], "solut": [0, 8, 24], "onli": [0, 7, 8, 11, 12, 14, 17, 24], "access": [0, 4, 18, 23], "within": [0, 7, 8, 9, 10, 11, 12, 15, 17], "sunet": [0, 12], "network": [0, 4, 14, 23], "vpn": 0, "outsid": 0, "link": [0, 5, 6, 7, 9], "you": [0, 4, 5, 6, 7, 10, 11, 12, 19, 21, 22], "credenti": 0, "all": [0, 4, 6, 7, 8, 9, 12, 17, 19, 23, 24], "swedish": [0, 14, 18, 22], "whole": [0, 17], "blue": 0, "hundr": 0, "virtual": [0, 24], "green": 0, "each": [0, 4, 7, 9, 14, 17, 23], "which": [0, 8, 9, 10, 12, 19, 24], "isol": 0, "transfer": [0, 11, 15], "through": [0, 4, 9, 11, 12, 14, 19, 23], "wharf": [0, 22], "special": 0, "file": [0, 4, 6, 7, 14, 17, 19, 23, 24], "area": [0, 11], "visibl": 0, "document": [0, 5, 17], "thinlinc": [0, 4, 14, 17, 22, 23], "softwar": [0, 4, 5, 6, 7, 8, 11, 12, 15, 17, 18, 22, 24], "log": [0, 4, 6, 7, 9, 11, 14, 17, 21, 22, 23], "command": [0, 5, 6, 7, 12, 17, 22], "line": [0, 5, 6, 9, 10, 17], "intro": [0, 9, 15, 23], "modul": [0, 5, 17, 22, 23, 24], "node": [0, 4, 5, 6, 7, 11, 12, 14, 19, 22, 23, 24], "packag": [0, 2, 17, 19, 24], "instal": [0, 5, 12, 15, 22, 24], "If": [0, 4, 7, 8, 9, 10, 11, 12, 17, 21], "handl": 0, "account": [0, 5, 9, 11, 12], "itself": 0, "ar": [0, 4, 7, 8, 10, 11, 12, 14, 15, 19, 21, 23, 24], "like": [0, 4, 7, 8, 9, 10, 11, 12, 14, 15, 17, 21, 23], "transit": [0, 23], "server": [0, 4, 8, 11, 12], "mani": [0, 4, 8, 9, 10, 17], "ask": [0, 8, 9, 10, 17, 21], "support": [0, 5, 8, 9, 10, 14, 17, 18, 22], "addit": [0, 8], "tool": [0, 2, 5, 6, 7, 8, 12, 22, 23], "number": [1, 4, 9, 11, 12, 17, 24], "reserv": [1, 4, 17], "project": [1, 5, 7, 9, 11, 15, 17, 18, 22, 23], "scilifelab": [1, 4, 15], "product": [1, 4, 14], "part": [1, 4, 7, 9, 10], "sen": [1, 4, 6, 7, 18, 22], "scienc": [1, 4, 14, 15], "storag": [1, 5, 14, 15, 22, 23], "total": [1, 4, 17, 24], "volum": [1, 4], "20": 1, "pb": [1, 4], "equival": [1, 4], "nearli": 1, "billion": 1, "3": [1, 4, 6, 7, 17, 23], "5": [1, 9, 20], "inch": 1, "floppi": 1, "disk": [1, 4, 14, 23, 24], "40": [1, 2, 4, 7, 10, 13], "000": [1, 4, 24], "year": [1, 2, 4], "128": [1, 4, 17, 24], "bit": [1, 2, 10], "encod": [1, 4], "music": [1, 4], "read": [1, 4, 8, 17, 24], "page": [1, 4, 5, 6, 9, 11, 12, 17, 21], "50": [2, 4, 7, 13], "theori": [2, 10, 14, 15, 17, 19], "jan": 2, "22": [2, 17], "23": 2, "onc": [2, 4, 8, 17], "fall": 2, "feb": 2, "13": [2, 17, 19], "focu": [2, 4], "slurm": [2, 5], "job": [2, 4, 5, 6, 7, 11, 12, 15, 24], "march": 2, "12": [2, 9, 17], "last": [2, 9, 10, 11, 17], "": [2, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 17, 18, 19, 23, 24], "nov": 2, "17": 2, "parallel": [2, 4, 6, 17, 23], "machin": [2, 9, 14, 18], "learn": [2, 8, 9, 11, 12, 18], "gpu": [2, 4, 6, 14, 17, 18, 23], "mai": [2, 6, 7, 8, 9, 10, 12, 17, 24], "june": 2, "date": [2, 14], "set": [2, 8, 9, 10, 11, 17, 24], "A": [2, 5, 7, 8, 14, 23, 24], "ml": [2, 17], "train": 2, "19": [2, 9], "lesson": 2, "upcom": 2, "event": 2, "codecademi": 2, "x": [2, 8, 9, 10, 11, 12, 17], "y": [2, 11, 12, 17], "villag": 2, "let": [4, 7, 10, 13, 15, 17, 19, 23, 24], "recap": [4, 15], "go": [4, 8, 9, 11, 12, 15, 17], "further": [4, 10, 17], "11": [4, 7, 10, 13, 15], "35": [4, 7, 15], "buzz": 4, "word": [4, 8, 15, 18], "rather": [4, 17], "narrow": [4, 17], "focus": [4, 18], "fast": [4, 8, 17], "calcul": [4, 6, 7, 11, 12, 14, 17, 23], "processor": [4, 14, 23], "throughput": 4, "larg": [4, 14, 17, 18, 22], "memori": [4, 17, 23], "demand": 4, "task": [4, 17], "here": [4, 5, 7, 11, 12, 14, 17, 23], "list": [4, 10, 17, 19], "three": [4, 9], "most": [4, 8, 9, 10, 17, 19], "common": [4, 5], "paradigm": 4, "float": 4, "point": [4, 9], "oper": [4, 8, 10], "per": [4, 17, 24], "second": 4, "flop": 4, "character": 4, "power": [4, 8, 17], "short": [4, 6, 9, 17, 19], "period": 4, "month": 4, "interest": [4, 6, 7, 21], "how": [4, 8, 9, 10, 11, 12, 14, 15, 17, 23], "complet": [4, 9, 10, 17], "over": [4, 17, 18, 19], "long": [4, 9, 14, 17, 18], "instead": [4, 9, 11, 12, 17], "independ": [4, 14], "sequenti": 4, "individu": [4, 6, 7, 14, 22], "schedul": [4, 17], "o": [4, 6, 8, 9, 11, 12, 18], "emphasi": 4, "accomplish": 4, "bridg": 4, "gap": 4, "reminisc": 4, "includ": [4, 9, 11, 12], "both": [4, 6, 11, 17, 24], "depend": [4, 12, 17, 24], "where": [4, 8, 11, 12, 17], "primari": 4, "metric": 4, "measur": 4, "mb": 4, "rate": 4, "oppos": [4, 17], "compris": 4, "multipl": [4, 17], "distinct": 4, "activ": [4, 17, 18, 19], "coupl": 4, "via": [4, 14, 23], "work": [4, 5, 6, 7, 8, 10, 11, 12, 14, 18, 19, 22, 23], "small": [4, 6, 7, 14, 23], "scale": [4, 14, 18, 22, 23], "raspberrypi": [4, 14, 23], "To": [4, 9, 14, 17, 19, 23], "supercomput": [4, 14, 15, 23], "rackham": [4, 9, 12, 14, 15, 16, 17, 18, 22, 24], "sever": [4, 12, 14, 17, 22, 23, 24], "core": [4, 14, 23], "ram": [4, 14, 17, 23], "local": [4, 9, 10, 11, 12, 14, 22, 23], "call": [4, 14, 17, 23], "scratch": [4, 14, 23], "login": [4, 5, 6, 7, 14, 17, 23, 24], "internet": [4, 14, 22, 23, 24], "ssh": [4, 11, 12, 14, 17, 23], "lighter": [4, 14, 23], "data": [4, 5, 6, 7, 8, 9, 10, 14, 15, 17, 18, 22, 23], "analysi": [4, 11, 12, 14, 17, 23], "ha": [4, 8, 12, 17, 24], "normal": [4, 8, 23], "one": [4, 8, 9, 11, 17, 23, 24], "util": [4, 23], "even": [4, 9, 11, 12, 17, 23], "keyword": [4, 9, 11, 23], "signal": [4, 23], "multi": [4, 23], "thread": [4, 17, 23], "mpi": [4, 17, 23], "distribut": [4, 15, 23], "openmp": [4, 17, 23], "share": [4, 7, 9, 23, 24], "run": [4, 5, 7, 8, 9, 11, 14, 18, 22, 23, 24], "start": [4, 6, 7, 8, 9, 10, 11, 12, 14, 17, 19, 23, 24], "interact": [4, 5, 6, 7, 8, 12, 22, 23, 24], "session": [4, 6, 7, 8, 9, 10, 11, 23, 24], "submit": [4, 6, 7, 11, 12, 17, 23], "batch": [4, 5, 22, 23, 24], "todai": [4, 5, 9], "introduct": [4, 6, 7, 17, 19, 22], "domu": [4, 23], "castor": [4, 23], "crex": [4, 23], "Its": [4, 23], "own": [4, 5, 6, 9, 11, 23], "2": [4, 6, 7, 9, 11, 12, 17, 19, 23], "tb": [4, 23], "re": [4, 8, 9, 10, 11], "write": [4, 7, 8, 9, 10, 17, 24], "copi": [4, 8, 10, 11, 17], "cp": [4, 8, 10], "myfil": 4, "snic_tmp": 4, "educ": [4, 15, 21], "miarka": [4, 14, 18, 19], "cloud": [4, 14, 21], "guid": [4, 7, 17, 19], "25": 4, "cygnu": 4, "ngi": [4, 14], "vulp": 4, "deliveri": 4, "gru": 4, "off": [4, 14], "load": [4, 14, 17, 19, 24], "lutra": 4, "found": [4, 17], "There": [4, 8, 10, 17], "appli": [4, 5, 9, 23], "supr": [4, 14], "gu": 4, "cours": [4, 8, 10, 11, 12, 15, 17, 21, 22, 24], "applic": [4, 5, 6, 7, 12, 14, 18], "done": [4, 8, 9, 17, 24], "suppos": 4, "rout": 4, "servic": [4, 14, 15, 18], "desk": 4, "detail": [4, 9, 14, 17], "wait": [4, 17], "its": [4, 8, 9, 17], "limit": [4, 9, 10, 11, 12, 17], "possibilit": 4, "determin": 4, "find": [5, 6, 7, 8, 9, 10, 17, 19, 24], "some": [5, 8, 9, 10, 12, 22, 24], "place": [5, 8, 9, 14, 22, 24], "present": [5, 6, 9, 10, 17, 19, 21, 24], "refer": [5, 7, 8], "lot": [5, 7, 8, 9, 17], "our": [5, 7, 11, 21], "new": [5, 9, 10, 11, 14, 24], "welcom": 5, "question": 5, "exercis": 5, "along": [5, 9, 10, 12, 19], "The": [5, 6, 9, 10, 14, 18, 19, 23], "topic": [5, 17], "icebreak": 5, "hackmd": 5, "mission": 5, "perform": [5, 15, 22, 23], "understand": [5, 22], "linux": [5, 9, 12, 22], "basic": [5, 6, 7, 8, 11, 22], "toolkit": 5, "cover": 5, "organis": [5, 6, 7, 14], "orient": [5, 6, 14], "staff": [5, 9], "expert": [5, 6, 7], "deal": 5, "htc": 5, "mtc": 5, "what": [5, 9, 10, 12], "hardwar": [5, 15], "databas": [5, 7], "script": [5, 8, 9, 10], "program": [5, 6, 7, 8, 10, 11, 12, 17, 24], "sbatch": 5, "queue": [5, 7, 22], "home": [5, 8, 9, 10, 11, 12, 23], "dir": [5, 9], "problem": [5, 17], "bianca": [5, 6, 7, 11, 12, 14, 15, 17, 18, 22, 24], "sensit": [5, 14, 15, 18, 22], "person": [5, 9, 21], "design": 5, "introductori": [5, 10], "center": [5, 14], "coderefineri": 5, "encc": 5, "eurocc": 5, "compet": [5, 15], "centr": [5, 15], "sweden": [5, 14, 15, 18], "extra": [5, 9], "content": [5, 10, 17], "q": [5, 7, 9], "move": [6, 7, 8, 10, 11], "being": [6, 7, 9], "novic": [6, 7], "naiss": [6, 7, 15, 22, 24], "wai": [6, 7, 9, 11, 12], "languag": [6, 7, 8, 17], "first": [6, 7, 8, 9, 10, 11, 12, 17, 19], "dai": [6, 7, 11, 12, 17], "github": [6, 7, 22], "websit": [6, 7], "reach": [6, 7, 11, 12, 17, 19], "main": [6, 7], "much": [6, 24], "relat": [6, 11], "talk": 6, "dure": [6, 17, 18], "best": [6, 12, 15, 24], "pose": 6, "directli": [6, 14], "voic": 6, "requir": [6, 11, 17, 19], "effort": 6, "explain": 6, "answer": 6, "io": [6, 7], "dianai": 6, "abl": [6, 7, 8, 9, 10, 17], "follow": [6, 7, 9], "built": [6, 7, 11, 12], "termin": [6, 7, 8, 9, 10, 12, 17, 19, 22, 24], "mac": [6, 7, 8, 12], "window": [6, 7, 8, 12], "powershel": [6, 7, 12], "prompt": [6, 7, 9, 10, 11, 12], "For": [6, 7, 8, 11, 12, 15, 17, 24], "benefit": [6, 7, 12], "see": [6, 7, 8, 9, 11, 17, 21, 24], "solv": [6, 7], "group": [6, 7, 9, 10], "classroom": 6, "left": 6, "menu": 6, "workshop": [6, 7, 10, 21], "pdf": [6, 8], "level": [6, 7, 9], "input": [6, 7, 17, 24], "output": [6, 7, 17], "pipe": [6, 7], "wildcard": [6, 7], "grep": [6, 7], "variabl": [6, 7, 24], "paramet": [6, 7], "filenam": [6, 7, 8, 9, 10, 17], "why": [6, 7], "IF": [6, 7], "statement": [6, 7], "test": [6, 7, 17, 23], "FOR": [6, 7], "while": [6, 7, 8], "loop": [6, 7], "background": [6, 7, 8, 10, 11, 12], "workload": [6, 7], "quick": [6, 17], "debug": [6, 17], "monitor": [6, 7], "modifi": [6, 9], "prioriti": [6, 17], "effici": [6, 15, 17], "workflow": [6, 17, 23], "snakemak": 6, "nextflow": 6, "flow": 6, "function": [6, 7, 18], "argument": 6, "object": 6, "bonu": [6, 24], "want": [7, 8, 9, 10, 17, 18], "skill": 7, "4": [7, 9, 17], "seri": 7, "alreadi": [7, 9, 11, 12, 17], "fundament": 7, "down": 7, "still": [7, 22, 24], "sure": [7, 9, 17, 24], "enjoi": 7, "tip": [7, 8, 9, 10, 17, 22, 24], "trick": 7, "henc": 7, "ve": [7, 9], "made": [7, 9, 11, 12], "sign": 7, "up": [7, 13, 17], "onlin": [7, 9], "chat": 7, "rais": 7, "hand": [7, 17], "breakout": 7, "room": [7, 22], "oss": 7, "queu": 7, "finish": [7, 17], "devel": [7, 17], "mode": [7, 8, 9], "structur": [7, 9], "execut": [7, 8, 9, 17], "creat": [7, 10, 11], "manner": 7, "so": [7, 9, 17], "00": [7, 17, 19], "55": [7, 12], "10": [7, 9, 10, 17], "c": [7, 8, 9, 10, 17, 24], "allow": [7, 11, 17, 24], "lunch": 7, "Then": [7, 11, 12], "do": [7, 8, 9, 10, 11, 12, 22, 24], "ani": [7, 9, 12, 15, 17, 24], "p2": 7, "summari": [7, 17], "look": [7, 17], "offici": 7, "webpag": 7, "tutori": 7, "anoth": [7, 9], "box": 7, "graph": 7, "code": [7, 12, 15, 17], "case": [7, 9], "usag": [7, 15, 17, 23, 24], "qaa": 7, "thing": [8, 9, 10, 17, 24], "awar": [8, 10], "daili": 8, "speak": 8, "unix": [8, 10], "compat": [8, 10], "kernel": 8, "sh": [8, 17], "ksh": 8, "csh": 8, "tcsh": 8, "actual": [8, 9, 17], "suppli": 8, "gnu": 8, "give": [8, 9, 11, 17, 24], "u": [8, 17, 21, 24], "name": [8, 17, 24], "controversi": 8, "en": 8, "wikipedia": 8, "org": [8, 11], "wiki": 8, "linux_naming_controversi": 8, "multius": 8, "base": [8, 18], "mean": [8, 9, 10, 11, 12, 17], "permiss": [8, 10], "secur": [8, 10, 11], "cento": 8, "redhat": 8, "often": [8, 9, 17, 19, 24], "default": [8, 9, 10, 17, 19, 24], "interfac": [8, 12, 17], "seen": [8, 9, 10], "becaus": [8, 17, 24], "sit": 8, "around": [8, 9, 10], "info": [8, 9, 10, 17], "word1": [8, 10], "word2": [8, 10], "word3": [8, 10], "configur": 8, "usual": [8, 9], "tell": [8, 19], "who": [8, 18], "bjornc": [8, 17], "rackham3": [8, 11, 12], "linux_tutori": [8, 9, 10, 11, 12], "chang": 8, "advanc": [8, 14, 18, 22], "doe": [8, 9, 17, 19], "matter": [8, 9, 10], "cyberc": 8, "biz": 8, "howto": 8, "setup": 8, "html": 8, "separ": 8, "space": [8, 9, 10, 24], "screen": [8, 9, 10, 23], "show": [8, 9], "whenev": [8, 10], "path": [8, 9, 10, 19], "strike": [8, 10], "kei": [8, 10], "habit": [8, 10], "save": [8, 9, 10, 11, 17], "hour": [8, 10, 21, 23], "gui": [8, 11], "mobaxterm": [8, 12], "keyboard": 8, "shortcut": 8, "shown": [8, 11], "cheatsheet": 8, "staffwww": 8, "fullcol": 8, "edu": 8, "sedward": 8, "usefulnanokeycommand": 8, "ctrl": [8, 10, 11, 17], "m": 8, "meta": 8, "alt": 8, "On": [8, 11, 14], "app": 8, "prefer": 8, "turn": 8, "option": [8, 9, 17], "suggest": [8, 24], "them": [8, 9, 17, 24], "difficulti": 8, "exit": [8, 17], "vim": [8, 10], "vi": 8, "insert": 8, "press": 8, "esc": 8, "cheat": 8, "sheet": 8, "coderwal": 8, "com": [8, 11, 22], "p": [8, 9, 17], "adv71w": 8, "gvim": 8, "featur": 8, "emac": [8, 10], "refcard": 8, "With": 8, "nw": 8, "keep": [8, 9, 17, 19], "version": [8, 11, 12, 17, 19, 24], "add": [8, 9, 17], "ex": [8, 9, 11], "z": [8, 9, 10, 11], "bg": [8, 10, 11], "send": [8, 17], "someth": [8, 10, 17], "interupt": [8, 10], "stuck": [8, 10], "paus": [8, 10], "continu": [8, 10], "foreground": [8, 10], "fg": [8, 10], "d": [8, 9, 10, 17, 19], "quit": [8, 9, 10, 17], "capit": [8, 10], "alwai": [8, 11, 12, 24], "don": [8, 11, 12, 17], "check": [8, 17, 24], "right": 8, "undo": [8, 10], "mv": [8, 10], "remov": [8, 9, 10], "rm": [8, 10, 24], "bewar": [8, 10], "overwrit": [8, 10], "delet": [8, 10], "wrong": [8, 10], "ones": [8, 10], "realli": [8, 9, 10, 17], "eras": [8, 9, 10], "alia": [8, 9, 10, 24], "overrid": [8, 9, 10], "f": [8, 9, 10], "bashrc": [8, 9, 10, 24], "directori": [8, 10, 17, 19, 23], "ad": [8, 9, 10], "everytim": [8, 10], "destroi": [8, 9, 10], "email": [8, 10, 15], "help": [8, 10], "dig": [9, 10], "import": [9, 10], "bash": [9, 10, 17, 24], "brows": [9, 10], "tar": [9, 10], "compress": [9, 10, 23], "extract": [9, 10], "physic": [9, 15, 17], "ignor": [9, 17], "got": 9, "hidden": [9, 19], "alpha": 9, "numer": 9, "order": [9, 21], "abov": [9, 11], "highlight": 9, "differ": [9, 12, 14], "color": 9, "lt": [9, 24], "sort": 9, "lrt": [9, 17, 24], "r": [9, 10, 11, 12, 17, 24], "revers": 9, "newest": 9, "almost": [9, 19], "scroll": 9, "arrow": 9, "target": [9, 10], "proj": [9, 10, 17, 19], "introtouppmax": [9, 10, 19], "lab": [9, 19], "try": [9, 10], "variou": 9, "tab": [9, 10, 11], "avoid": [9, 19, 24], "typo": 9, "figur": 9, "out": [9, 17, 22], "goe": 9, "back": 9, "hierarchi": 9, "sourc": [9, 24], "well": [9, 24], "didn": 9, "t": [9, 10, 11, 12, 17], "error": [9, 11, 12, 22, 24], "sai": 9, "recurs": 9, "subdirectori": [9, 10], "just": [9, 17, 24], "newfil": 9, "same": [9, 10, 17], "copyfil": 9, "maco": 9, "upload": [9, 11], "bob": 9, "macbook": 9, "myinput": 9, "copyofmyinput": 9, "download": [9, 11], "mydata": 9, "copyofmydata": 9, "renamedfil": 9, "gz": 9, "tare": 9, "gzip": 9, "vxzf": 9, "v": [9, 23], "erbos": 9, "tract": 9, "ip": 9, "ilenam": 9, "should": [9, 11, 12, 17], "czfv": 9, "edit": [9, 10], "everi": 9, "this_is_empti": 9, "hmmmm": 9, "this_has_a_fil": 9, "again": 9, "nobodi": 9, "rememb": [9, 17], "whether": 9, "choos": 9, "forc": 9, "action": 9, "search": 9, "n": [9, 17], "forward": [9, 12, 24], "backward": 9, "scan": 9, "hit": 9, "spend": 9, "many_fil": 9, "txt": 9, "file_1": 9, "1": [9, 14, 17, 24], "docx": 9, "clean": 9, "temporari": 9, "tmp": 9, "nb": 9, "could": [9, 12, 17], "wise": [9, 24], "In": [9, 10, 11, 15, 21], "wish": 9, "old_project": [9, 10], "hmm": 9, "dump": 9, "text": [9, 10], "the_best": 9, "yummi": 9, "concaten": 9, "wizardri": 9, "combinedfil": 9, "written": [9, 17], "append": 9, "handi": 9, "doesn": 9, "drwxrwxr": 9, "marcusl": 9, "4096": 9, "sep": 9, "2012": 9, "external_hdd": 9, "rwxr": 9, "xr": 9, "17198": 9, "jul": 9, "16": [9, 17], "lead": 9, "symbol": 9, "regular": 9, "tomorrow": 9, "exist": [9, 19], "rwx": 9, "ead": 9, "w": 9, "rite": 9, "ecut": 9, "xyz": 9, "elsewher": 9, "everyon": 9, "els": 9, "liter": 9, "thousand": 9, "stranger": 9, "travers": 9, "huh": [9, 10], "rwxrwsr": [9, 10], "gid": [9, 10], "roup": [9, 10], "id": [9, 10, 12, 17], "creator": [9, 10], "launcher": [9, 10], "sticki": [9, 10], "behaviour": [9, 10], "group_id": [9, 10], "inherit": [9, 10], "parent": [9, 10], "rare": [9, 10], "instanc": [9, 10, 11, 12, 17], "p_introtouppmax": [9, 10], "accid": 9, "me": 9, "non": 9, "member": [9, 24], "form": [9, 15], "whom": 9, "owner": 9, "certain": 9, "AND": [9, 18], "equal": 9, "caus": [9, 24], "unment": 9, "except": [9, 19], "affect": 9, "wrote": 9, "warn": [9, 11, 12], "kept": 9, "rw": 9, "xw": 9, "put": [9, 11, 17], "take": [9, 10], "sub": 9, "therein": 9, "come": 9, "across": [9, 19], "digit": 9, "correspond": 9, "respect": [9, 15], "7": [9, 17], "0": [9, 17], "would": 9, "6": [9, 17], "old": [9, 10], "wouldn": [9, 10], "lose": [9, 10], "imagin": [9, 10], "important_result": [9, 10], "last_years_data": [9, 10], "those": [9, 10, 17, 24], "moment": [9, 10], "plai": [9, 10, 17], "explor": [9, 10, 18], "effect": [9, 10, 17], "wr": [9, 10], "tuesdai": [9, 10], "wednesdai": [9, 10, 17], "editor": 10, "getting_start": [10, 12, 13], "These": [10, 19], "typic": [10, 24], "exampl": [10, 11, 12, 17, 24], "nano": 10, "graphic": [10, 17], "gedit": [10, 11], "pwd": [10, 17], "l": [10, 17, 24], "cd": [10, 17, 19], "mkdir": 10, "scp": [10, 12], "remot": [10, 12], "rmdir": 10, "empti": 10, "cat": [10, 17], "print": 10, "head": 10, "tail": 10, "less": [10, 24], "man": 10, "linux_bas": 10, "flag": [10, 17], "arriv": [11, 12], "specif": [11, 12, 17, 19], "yourself": [11, 12], "lack": [11, 12], "visit": [11, 12], "folder": [11, 12, 17, 24], "light": [11, 12], "clusternam": [11, 12], "low": [11, 12, 17, 24], "willl": 11, "ater": 11, "state": [11, 12], "singl": [11, 12, 17], "letter": [11, 12], "r484": [11, 12], "xquartz": [11, 12], "enabl": [11, 17], "usernam": [11, 12, 17, 24], "sftp": [11, 12], "protocol": 11, "client": [11, 12], "filezilla": [11, 12], "choic": 11, "php": 11, "suffici": [11, 12, 17], "shell": [11, 12], "frame": 11, "section": [11, 12], "below": [11, 17], "putti": [11, 12], "No": [11, 22], "Or": 11, "better": 11, "texteditor": 11, "renam": 11, "bookmark": 11, "winscp": [11, 12], "net": 11, "eng": 11, "sent": [11, 17], "connect": [11, 14, 22, 24], "primarili": 11, "secondari": 11, "mention": 11, "lock": 11, "end": [11, 17, 21], "xey": [11, 12], "altern": [11, 17, 19], "sleep": 11, "receiv": [11, 12], "offer": [11, 18], "desktop": 11, "factor": 11, "authent": 11, "cendio": 11, "either": [11, 12, 17], "my": 12, "9": [12, 14, 17], "demo": 12, "plan": 12, "insid": 12, "login_rackham": 12, "x11": [12, 19], "iterm2": 12, "cmd": 12, "mutipl": 12, "ftp": 12, "browser": 12, "cyberduck": 12, "subsystem": 12, "wsl": 12, "integr": [12, 23], "develop": [12, 17, 18, 19, 24], "visual": 12, "studio": 12, "sophist": 12, "rstudio": 12, "matlab": [12, 19], "etcetera": [12, 19], "extens": [12, 19], "somewhat": 12, "easier": 12, "step": 13, "project_appli": 13, "uppmac": 14, "uppsala": [14, 15, 21, 22], "multidisciplinari": 14, "academ": [14, 15], "infrastructur": [14, 15], "qualiti": [14, 15], "januari": 14, "2023": 14, "organ": [14, 15], "replac": 14, "link\u00f6p": [14, 18], "host": [14, 15], "act": 14, "perspect": 14, "respons": [14, 17], "initi": 14, "minim": 14, "regim": 14, "faq": 14, "fund": [14, 15], "afternoon": [14, 17], "purpos": [14, 15, 17, 18], "snowi": [14, 15, 17, 18, 22], "sequenc": [14, 15], "pre": 14, "befor": [14, 17, 24], "thei": [14, 17, 19, 24], "deliv": [14, 23], "anymor": 14, "di": [14, 18], "region": [14, 18], "east": [14, 18], "avail": [14, 19, 22, 24], "capac": [14, 17, 22], "universtii": [14, 22], "consist": [14, 22], "inter": [14, 17, 22], "togeth": [14, 22], "littl": 15, "deeper": 15, "vr": 15, "src": 15, "consortium": 15, "unit": [15, 17], "nbi": 15, "bioinformat": [15, 17, 19], "relev": 15, "driven": 15, "life": [15, 18], "especi": 15, "know": 15, "field": 15, "chemistri": 15, "geoscienc": 15, "troubleshoot": 15, "advic": 15, "possibli": 15, "queri": 15, "ticket": 15, "accord": [15, 17], "wed": 17, "otherwis": [17, 24], "naiss2024": 17, "49": 17, "two": 17, "worker": 17, "hard": 17, "1000": 17, "300": [17, 24], "5000": 17, "defin": [17, 24], "therefor": 17, "happen": 17, "book": 17, "describ": 17, "definit": 17, "navig": 17, "collect": 17, "serial": 17, "ONE": 17, "occupi": 17, "cpu": [17, 18], "than": 17, "gb": [17, 24], "112": 17, "mandatori": 17, "partit": 17, "larger": [17, 18], "devcor": 17, "axi": 17, "hole": 17, "fit": 17, "immedi": 17, "wide": 17, "too": [17, 24], "five": 17, "easiest": 17, "howev": 17, "But": 17, "overal": 17, "shorter": 17, "wast": 17, "charg": 17, "60": 17, "120": 17, "full": [17, 19, 24], "192": 17, "simul": 17, "bound": 17, "mainli": 17, "messag": 17, "pass": 17, "seldom": 17, "bottleneck": 17, "duplic": 17, "fat": 17, "wall": 17, "quickli": 17, "logout": 17, "number_of_cor": 17, "session_dur": 17, "8": 17, "durat": 17, "recommend": 17, "least": 17, "squeue": [17, 24], "r483": 17, "ongo": 17, "begun": 17, "had": 17, "hostnam": 17, "srun": 17, "resembl": 17, "r_packag": 17, "stick": 17, "2022": 17, "07": 17, "554": 17, "slow": 17, "workspac": 17, "bin": 17, "top": 17, "practic": [17, 24], "reload": 17, "fresh": 17, "interfer": 17, "ordinari": 17, "interpret": 17, "jobscript": 17, "past": 17, "ten": 17, "j": 17, "template_script": 17, "bioinfo": [17, 19], "echo": 17, "hello": 17, "world": 17, "mem256gb": 17, "256": 17, "mem512gb": 17, "512": 17, "gre": 17, "01": 17, "240": 17, "restrict": [17, 23], "forbid": 17, "draft": 17, "anywai": 17, "free": 17, "jobinfo": 17, "finishedjobinfo": 17, "jobstat": [17, 23], "eog": 17, "watch": 17, "png": 17, "bianca_combined_jobinfo": 17, "priorit": 17, "fewer": 17, "ineffici": 17, "irregularli": 17, "regist": 17, "plot": [17, 23], "interactic": 17, "discov": 17, "jobtempl": 17, "chmod": 17, "ran": 17, "correctli": 17, "manual": 17, "mostli": 18, "intel": 18, "nvidia": 18, "toward": 18, "dedic": 18, "pdc": 18, "kth": 18, "dardel": 18, "Not": 18, "crai": 18, "amd": 18, "nsc": 18, "tetralith": 18, "hpc2n": 18, "ume\u00e5": 18, "kebnekais": 18, "c3se": 18, "chalmer": 18, "alvi": 18, "acceler": 18, "ai": 18, "techniqu": 18, "lumi": 18, "finland": 18, "aim": 18, "lunarc": 18, "lund": 18, "aurora": 18, "vera": 18, "berzeliu": 18, "complement": 18, "flexibl": 18, "recent": 18, "technologi": 18, "iot": 18, "big": 18, "apach": 18, "hadoop": 18, "spark": 18, "haze": 18, "introduc": 18, "octob": 18, "2017": [18, 24], "upgrad": 18, "2020": 18, "north": 18, "west": 18, "grant": 18, "committe": 18, "snac": 18, "known": 18, "800": 19, "chao": 19, "collis": 19, "explicitli": 19, "efault": 19, "latest": 19, "demark": 19, "reproduc": 19, "r2020b": 19, "samtool": 19, "cannot": [19, 24], "request": 19, "sunrai": 19, "locat": 19, "central": [19, 23], "throughout": 19, "python": [19, 24], "18": 20, "24": 20, "teach": 21, "result": 21, "paper": 21, "confer": 21, "kindli": 21, "acknowledg": 21, "direct": 22, "under": 22, "constructon": 22, "miss": 22, "anyth": 22, "spot": 22, "issu": 22, "detach": 23, "sge": 23, "debugg": 23, "profil": 23, "luigi": 23, "lifescop": 23, "viewer": 23, "igv": 23, "galaxi": 23, "cram": 23, "bam": 23, "expand": 24, "helper": 24, "32": 24, "size": 24, "attach": 24, "uquota": 24, "current": 24, "projinfo": 24, "334": 24, "specifi": 24, "naiss2017": 24, "conda": 24, "clash": 24, "forgotten": 24, "unexeptect": 24, "unexpect": 24, "fill": 24, "conflict": 24, "alias": 24, "bj\u00f6rn": 24, "sq": 24, "sc": 24, "scancel": 24, "mix": 24, "nice": 24, "eas": 24, "environemnt": 24, "virtualenv": 24, "straight": 24, "reason": 24, "teacher": 24, "stuff": 24, "othersid": 24, "easili": 24, "becom": 24, "public": 24, "count": 24}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"naiss": [0, 2, 5, 13, 14], "sen": [0, 5], "bianca": [0, 1, 2, 4, 23], "object": [0, 4, 8, 9, 10, 11, 12, 13, 14, 15, 17, 19, 24], "The": [0, 1, 4, 7, 8, 11, 12, 15, 17, 22], "workshop": [0, 2, 5], "instructor": [0, 2, 4, 10, 12, 13, 14, 15, 17, 19, 24], "note": [0, 2, 4, 10, 12, 13, 14, 15, 17, 19, 24], "sensit": 0, "person": 0, "data": [0, 2], "appli": [0, 21], "project": [0, 4, 13, 21, 24], "": [0, 9, 22], "design": 0, "ha": 0, "internet": 0, "introduct": [0, 2, 5], "cours": [0, 2, 5, 6, 7], "keypoint": [0, 4, 8, 10, 11, 12, 14, 17, 19, 24], "cluster": [1, 4, 14, 17, 18, 23, 24], "rackham": [1, 11, 23], "snowi": [1, 4, 23], "miarka": 1, "uppmax": [1, 2, 4, 5, 6, 7, 14, 15, 21, 24], "cloud": [1, 2, 18], "system": [1, 4, 9, 14, 18, 23], "usag": [1, 4], "introductori": 2, "linux": [2, 6, 7, 8, 10, 11, 23], "intro": [2, 5, 6, 7], "thi": 2, "one": 2, "awk": 2, "other": [2, 17], "basic": [2, 4, 9, 10, 23], "singular": 2, "run": [2, 17, 19], "r": [2, 20], "python": [2, 6, 7, 20], "julia": [2, 20], "hpc": [2, 4, 14, 18, 23], "git": 2, "hpc2n": 2, "program": [2, 9, 19], "formal": 2, "snic": [2, 18], "center": 2, "coderefineri": 2, "encc": 2, "eurocc": 2, "nation": 2, "compet": 2, "centr": 2, "sweden": 2, "link": [2, 17], "extra": 2, "materi": [2, 6, 7], "manag": [2, 9, 10, 11], "seminar": 2, "gener": [2, 11, 12, 23], "bioinformat": 2, "swedish": 2, "scienc": [2, 18, 23], "exercis": [3, 6, 7, 9, 12, 17], "high": [4, 14], "perform": [4, 14, 24], "comput": [4, 14, 17], "htc": 4, "mtc": 4, "what": [4, 8, 14, 17, 23, 24], "i": [4, 6, 7, 8, 14, 17, 23], "storag": [4, 18, 21], "hardwar": 4, "name": [4, 9], "home": [4, 17, 24], "directori": [4, 9], "A": [4, 17, 22], "littl": 4, "bit": [4, 9], "more": [4, 5, 7], "about": [4, 5], "first": 5, "dai": 5, "lesson": [5, 6, 7], "overview": [5, 14, 15, 24], "get": [5, 9, 21], "start": [5, 21], "us": [5, 8, 11, 17, 19, 23], "summari": [5, 22], "welcom": [6, 7], "site": 6, "punchlin": [6, 7], "learn": [6, 7, 20], "outcom": [6, 7], "question": [6, 7, 8, 10, 11, 12, 14], "type": [6, 7, 17], "along": [6, 7, 17], "exampl": [6, 8, 9], "arrang": 6, "your": [6, 11], "screen": 6, "modul": [6, 19], "9": [6, 7], "12": [6, 7], "mondai": [6, 7], "13": [6, 7], "16": [6, 7], "ii": [6, 7], "tuesdai": [6, 7], "iii": [6, 7], "bash": [6, 7, 8], "script": [6, 7, 17, 19], "wednesdai": [6, 7], "slurm": [6, 7, 17, 23], "thursdai": [6, 7], "icebreak": [6, 7], "hackmd": [6, 7], "line": [7, 8, 11], "aim": 7, "new": [7, 22], "document": [7, 10, 12, 13, 22], "page": 7, "veri": 7, "much": 7, "relat": 7, "present": 7, "talk": 7, "dure": 7, "requir": 7, "effort": 7, "explain": 7, "answer": [7, 17], "topic": 7, "todai": [7, 8, 22], "preliminari": 7, "schedul": 7, "style": [7, 9], "lectur": 7, "come": 8, "differ": [8, 17], "distribut": 8, "dialect": 8, "sai": 8, "flavour": 8, "command": [8, 9, 10, 11], "bourn": 8, "again": [8, 24], "shell": [8, 9], "prompt": 8, "tab": 8, "complet": 8, "edit": 8, "file": [8, 9, 10, 11], "text": 8, "editor": 8, "gedit": 8, "nano": 8, "Not": 8, "try": [8, 11, 12, 17], "you": [8, 9, 17, 24], "haven": 8, "t": 8, "befor": 8, "graphic": [8, 11, 12], "typic": 8, "sourc": 8, "error": 8, "caution": 8, "toolkit": [9, 10], "we": [9, 10], "cover": [9, 10], "navig": [9, 10], "read": [9, 10], "chang": [9, 10], "properti": [9, 10], "pwd": 9, "where": 9, "ar": [9, 17], "now": 9, "print": 9, "current": 9, "work": [9, 17, 24], "l": 9, "list": 9, "content": [9, 22], "cd": 9, "experi": 9, "solut": [9, 10, 12, 17, 19], "copi": 9, "creat": 9, "move": 9, "mkdir": 9, "make": 9, "cp": 9, "scp": 9, "secur": 9, "remot": 9, "mv": 9, "renam": 9, "archiv": 9, "delet": 9, "rm": 9, "rmdir": 9, "an": [9, 17], "empti": 9, "help": 9, "man": 9, "manual": 9, "look": 9, "up": [9, 21], "right": 9, "flag": 9, "let": 9, "wild": 9, "wildcard": 9, "cat": 9, "con": 9, "ins": 9, "enat": 9, "standard": 9, "output": 9, "head": 9, "displai": 9, "top": 9, "u": [9, 20], "ing": 9, "tail": 9, "end": 9, "less": 9, "whole": 9, "permiss": 9, "For": [9, 10], "interest": [9, 10], "common": [9, 24], "issu": 9, "syntax": 9, "chmod": [9, 10], "755": 9, "binari": 9, "sum": 9, "octal": 9, "mask": 9, "hand": [9, 10, 19], "see": [10, 12, 13], "login": [11, 12], "mac": 11, "user": [11, 18, 20, 21, 23], "termin": 11, "problem": [11, 24], "instal": [11, 19, 23], "window": 11, "most": 11, "straight": 11, "forward": 11, "mobaxterm": 11, "x11": 11, "from": [11, 12], "thinlinc": [11, 12], "all": 11, "platform": 11, "web": [11, 12], "understand": 12, "procedur": 12, "log": 12, "ternmin": 12, "enabl": 12, "mission": [14, 15], "organis": 15, "orient": 15, "support": 15, "staff": 15, "applic": 15, "expert": 15, "deal": 15, "alloc": 15, "account": [15, 21], "node": 17, "sbatch": 17, "job": [17, 23], "queue": 17, "some": 17, "keyword": 17, "paramet": 17, "core": [17, 24], "hour": [17, 24], "choic": 17, "kind": [17, 24], "do": 17, "cheat": 17, "sheet": 17, "walltim": 17, "interact": 17, "rstudio": 17, "batch": 17, "simpl": 17, "templat": 17, "tool": [17, 19], "why": 17, "alwai": 17, "session": 17, "which": 17, "approach": 17, "best": 17, "follow": 17, "case": 17, "local": 18, "financ": 18, "ssc": 18, "swestor": 18, "train": 18, "softwar": [19, 20, 23], "databas": 19, "own": 19, "demo": 19, "fortran": 19, "specif": [20, 23], "guid": [20, 21, 23], "allinea": 20, "ddt": 20, "gamess": 20, "garli": 20, "gaussian": 20, "gromac": 20, "matlab": 20, "molca": 20, "nvidia": 20, "deep": 20, "framework": 20, "perl": 20, "packag": 20, "tensorflow": 20, "wrf": 20, "set": 21, "two": 21, "factor": 21, "authent": 21, "cpu": 21, "time": 21, "polici": 21, "instruct": 21, "disk": 21, "q": 22, "faq": 23, "submiss": 23, "life": 23, "dir": 24, "quota": 24, "discuss": 24}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 57}, "alltitles": {"NAISS SENS and Bianca": [[0, "naiss-sens-and-bianca"]], "Objectives": [[0, "objectives-0"], [4, "objectives-0"], [8, "objectives-0"], [9, "objectives-0"], [10, "objectives-0"], [10, "objectives-1"], [11, "objectives-0"], [12, "objectives-0"], [13, "objectives-0"], [14, "objectives-0"], [15, "objectives-0"], [17, "objectives-0"], [19, "objectives-0"], [24, "objectives-0"]], "The Bianca workshop": [[0, null]], "Instructor note": [[0, "instructor-note-0"], [2, "instructor-note-0"], [4, "instructor-note-0"], [10, "instructor-note-0"], [10, "instructor-note-1"], [12, "instructor-note-0"], [13, "instructor-note-0"], [14, "instructor-note-0"], [15, "instructor-note-0"], [17, "instructor-note-0"], [19, "instructor-note-0"], [24, "instructor-note-0"]], "Sensitive personal data": [[0, "sensitive-personal-data"]], "Apply for project": [[0, "apply-for-project"]], "Bianca": [[0, "bianca"], [1, "bianca"]], "Bianca\u2019s design": [[0, "biancas-design"]], "Bianca has no Internet": [[0, "bianca-has-no-internet"]], "Introduction course": [[0, "introduction-course"]], "Keypoints": [[0, "keypoints-0"], [4, "keypoints-0"], [8, "keypoints-0"], [10, "keypoints-0"], [11, "keypoints-0"], [12, "keypoints-0"], [14, "keypoints-0"], [17, "keypoints-0"], [19, "keypoints-0"], [24, "keypoints-0"]], "Clusters": [[1, "clusters"], [4, "clusters"]], "Rackham": [[1, "rackham"]], "Snowy": [[1, "snowy"]], "Miarka": [[1, "miarka"]], "The UPPMAX Cloud": [[1, "the-uppmax-cloud"]], "System usage": [[1, "system-usage"], [4, "system-usage"]], "Courses and workshops": [[2, "courses-and-workshops"], [5, null]], "Introductory Linux courses": [[2, "introductory-linux-courses"]], "Intro to UPPMAX (This one)": [[2, "intro-to-uppmax-this-one"]], "Awk workshop": [[2, "awk-workshop"]], "Other UPPMAX courses": [[2, "other-uppmax-courses"]], "Intro to Bianca": [[2, "intro-to-bianca"]], "Basic Singularity": [[2, "basic-singularity"]], "Introduction to running R, Python, and Julia in HPC": [[2, "introduction-to-running-r-python-and-julia-in-hpc"]], "Git course": [[2, "git-course"]], "Python at UPPMAX and HPC2N": [[2, "python-at-uppmax-and-hpc2n"]], "Programming formalisms": [[2, "programming-formalisms"]], "Courses at other NAISS/SNIC Centers": [[2, "courses-at-other-naiss-snic-centers"]], "CodeRefinery": [[2, "coderefinery"]], "ENCCS (EuroCC National Competence Centre Sweden": [[2, "enccs-eurocc-national-competence-centre-sweden"]], "Links to extra material": [[2, "links-to-extra-material"]], "Data Management Seminar": [[2, "data-management-seminar"]], "Python": [[2, "python"], [20, "python"]], "General Bioinformatics": [[2, "general-bioinformatics"]], "Swedish Science Cloud": [[2, "swedish-science-cloud"]], "Exercises": [[3, "exercises"], [6, "exercises"], [7, "exercises"], [12, "exercises"], [17, "exercises"]], "High Performance Computing \u2014 HPC": [[4, "high-performance-computing-hpc"], [14, "high-performance-computing-hpc"]], "HPC, HTC and MTC": [[4, "hpc-htc-and-mtc"]], "What is a cluster?": [[4, "what-is-a-cluster"], [14, "what-is-a-cluster"], [23, "what-is-a-cluster"]], "Storage basics": [[4, "storage-basics"]], "The UPPMAX hardware": [[4, "the-uppmax-hardware"]], "UPPMAX storage system names (projects & home directories)": [[4, "uppmax-storage-system-names-projects-home-directories"]], "A little bit more about Snowy": [[4, "a-little-bit-more-about-snowy"]], "About Bianca?": [[4, "about-bianca"]], "Introduction to UPPMAX \u2014 first day lessons": [[5, "introduction-to-uppmax-first-day-lessons"]], "Intro": [[5, null]], "Overview": [[5, null], [14, "overview"], [15, "overview"]], "Get started": [[5, null]], "More about UPPMAX": [[5, null]], "Using UPPMAX": [[5, null]], "NAISS SENS": [[5, null]], "Summary": [[5, null], [22, "summary"]], "Welcome on-site!": [[6, "welcome-on-site"]], "Course punchline": [[6, null], [7, null]], "Learning outcomes": [[6, null], [7, null]], "Lesson material": [[6, "lesson-material"], [7, "lesson-material"]], "Questions": [[6, "questions"], [7, "questions"], [8, "questions-0"], [10, "questions-0"], [11, "questions-0"], [12, "questions-0"], [14, "questions-0"]], "Type along": [[6, "type-along"], [7, "type-along"]], "Example of arrangement on your screen": [[6, "example-of-arrangement-on-your-screen"]], "Lesson Modules": [[6, "lesson-modules"]], "Linux I at 9-12 Monday": [[6, "linux-i-at-9-12-monday"], [7, "linux-i-at-9-12-monday"]], "Intro to UPPMAX 13-16 Monday": [[6, "intro-to-uppmax-13-16-monday"], [7, "intro-to-uppmax-13-16-monday"]], "Linux II\u00a09-12 Tuesday": [[6, "linux-ii-9-12-tuesday"], [7, "linux-ii-9-12-tuesday"]], "Linux III 13-16 Tuesday": [[6, "linux-iii-13-16-tuesday"], [7, "linux-iii-13-16-tuesday"]], "Bash Scripts\u00a09-12 Wednesday": [[6, "bash-scripts-9-12-wednesday"], [7, "bash-scripts-9-12-wednesday"]], "Slurm at UPPMAX 13-16 Wednesday": [[6, "slurm-at-uppmax-13-16-wednesday"], [7, "slurm-at-uppmax-13-16-wednesday"]], "Python I\u00a09-12 Thursday": [[6, "python-i-9-12-thursday"], [7, "python-i-9-12-thursday"]], "Python II 13-16 Thursday": [[6, "python-ii-13-16-thursday"], [7, "python-ii-13-16-thursday"]], "Icebreaker in HackMD": [[6, "icebreaker-in-hackmd"], [7, "icebreaker-in-hackmd"]], "Welcome on-line!": [[7, "welcome-on-line"]], "Course aim": [[7, null]], "New documentation page": [[7, "new-documentation-page"]], "Questions very much related to the present talk during the lessons": [[7, "questions-very-much-related-to-the-present-talk-during-the-lessons"]], "Questions requiring more effort to explain or answer": [[7, "questions-requiring-more-effort-to-explain-or-answer"]], "The topics": [[7, "the-topics"]], "Today": [[7, "today"]], "Preliminary schedule": [[7, "preliminary-schedule"]], "The New documentation": [[7, "the-new-documentation"]], "Style of todays lectures": [[7, "style-of-todays-lectures"]], "Linux": [[8, "linux"]], "What is Linux?": [[8, "what-is-linux"]], "Linux comes in different distributions, dialects or, say, flavours.": [[8, "linux-comes-in-different-distributions-dialects-or-say-flavours"]], "Using the command line": [[8, "using-the-command-line"]], "Command line with bash (Bourne Again Shell)": [[8, "command-line-with-bash-bourne-again-shell"]], "The prompt": [[8, "the-prompt"]], "Example bash command": [[8, "example-bash-command"]], "Tab Completion": [[8, "tab-completion"]], "Editing files with file/text editors": [[8, "editing-files-with-file-text-editors"]], "gedit": [[8, "gedit"]], "nano": [[8, "nano"]], "Not to try today if you haven\u2019t used before!!!": [[8, "solution-0"]], "The graphical editors": [[8, "the-graphical-editors"]], "Try gedit": [[8, "exercise-0"]], "Typical sources of error": [[8, "typical-sources-of-error"]], "Caution!!": [[8, "caution"]], "Basic toolkit": [[9, "basic-toolkit"], [10, "basic-toolkit"]], "We will cover these commands": [[9, "we-will-cover-these-commands"], [10, "we-will-cover-these-commands"]], "Navigation and file management": [[9, "navigation-and-file-management"], [10, "navigation-and-file-management"]], "Read files and change file properties": [[9, "read-files-and-change-file-properties"], [10, "read-files-and-change-file-properties"]], "File system Navigation": [[9, "file-system-navigation"]], "pwd \u2014 where are you now? \u201cPrint name of current/Working Directory\u201d": [[9, "pwd-where-are-you-now-print-name-of-current-working-directory"]], "ls \u2014 list directory contents": [[9, "ls-list-directory-contents"]], "cd \u2014 Change the shell working Directory": [[9, "cd-change-the-shell-working-directory"]], "Experiment with cd": [[9, "exercise-0"]], "Solution": [[9, "solution-0"], [9, "solution-1"], [9, "solution-3"], [9, "solution-5"], [10, "solution-0"], [12, "solution-0"], [17, "solution-0"], [19, "solution-0"]], "Copy, Create, Move": [[9, "copy-create-move"]], "mkdir \u2014 make directories": [[9, "mkdir-make-directories"]], "cp \u2014 copy files and directories": [[9, "cp-copy-files-and-directories"]], "scp \u2014secure copy (remote file copy program)": [[9, "scp-secure-copy-remote-file-copy-program"]], "mv \u2014 move/rename file": [[9, "mv-move-rename-file"]], "Archiving": [[9, "archiving"]], "Deleting": [[9, "deleting"]], "rm \u2014 delete files or directories": [[9, "rm-delete-files-or-directories"]], "rmdir \u2014 delete an empty directory": [[9, "rmdir-delete-an-empty-directory"]], "Help": [[9, "help"]], "man \u2014 manual, look up the right flags": [[9, "man-manual-look-up-the-right-flags"]], "Exercise": [[9, "exercise-1"], [9, "exercise-2"]], "Let\u2019s get wild with Wildcards": [[9, "lets-get-wild-with-wildcards"]], "Reading files": [[9, "reading-files"]], "cat - concatenate files and print on the standard output": [[9, "cat-concatenate-files-and-print-on-the-standard-output"]], "head \u2014 display the top (heading) of a file": [[9, "head-display-the-top-heading-of-a-file"]], "tail \u2014 display the end of a file": [[9, "tail-display-the-end-of-a-file"]], "less \u2014 read a whole file": [[9, "less-read-a-whole-file"]], "File permissions": [[9, "file-permissions"]], "Example": [[9, "example"]], "For the interested": [[9, "solution-2"], [10, "solution-1"]], "Changing permissions": [[9, "changing-permissions"]], "Common issues": [[9, null]], "Syntax": [[9, "syntax"]], "Examples": [[9, "examples"]], "chmod 755 style - binary sum - \u201coctal bit mask\u201d": [[9, "solution-4"]], "chmod \u2014 Hands-on": [[9, "exercise-3"], [10, "exercise-0"]], "Linux and basic toolkit": [[10, "linux-and-basic-toolkit"]], "See the documentation": [[10, null], [10, null], [12, null], [13, null]], "Login": [[11, "login"], [12, "login"]], "The login": [[11, "the-login"], [12, "the-login"]], "MAC and LINUX users": [[11, "mac-and-linux-users"]], "Terminal": [[11, "terminal"]], "Graphical file manager": [[11, "graphical-file-manager"]], "Problems with installations?": [[11, "callout-0"], [11, "callout-1"]], "Windows users": [[11, "windows-users"]], "Most straight-forward": [[11, "most-straight-forward"]], "MobaXterm": [[11, "mobaxterm"]], "Graphical file manager (Windows)": [[11, "graphical-file-manager-windows"]], "X11-forwarding from the command line (generally)": [[11, "x11-forwarding-from-the-command-line-generally"]], "Login to Rackham, using your terminal": [[11, "exercise-0"]], "ThinLinc (all platforms!)": [[11, "thinlinc-all-platforms"]], "Try Thinlinc from web": [[11, "exercise-1"], [12, "exercise-2"]], "General understanding": [[12, "general-understanding"]], "Login procedure": [[12, "discussion-0"], [12, null]], "Log in with a ternminal": [[12, "exercise-0"]], "Log in with a ternminal but enable graphics": [[12, "exercise-1"]], "NAISS and projects": [[13, "naiss-and-projects"]], "NAISS": [[14, "naiss"]], "UPPMAX missions": [[14, "uppmax-missions"], [15, "uppmax-missions"]], "UPPMAX systems": [[14, "uppmax-systems"]], "Overview of the UPPMAX systems": [[14, "overview-of-the-uppmax-systems"]], "Organisational orienteering": [[15, "organisational-orienteering"]], "UPPMAX support staff": [[15, "uppmax-support-staff"]], "The Application experts not dealing with allocation and accounts": [[15, "the-application-experts-not-dealing-with-allocation-and-accounts"]], "Using the compute nodes": [[17, "using-the-compute-nodes"]], "The compute nodes": [[17, "the-compute-nodes"]], "Slurm, sbatch, the job queue": [[17, "slurm-sbatch-the-job-queue"]], "Jobs": [[17, "jobs"]], "\u201cSome keywords\u201d": [[17, null]], "Slurm parameters": [[17, "slurm-parameters"]], "The queue": [[17, "the-queue"]], "Core-hours": [[17, "core-hours"]], "Choices": [[17, "choices"]], "What kind of compute work are you doing?": [[17, "what-kind-of-compute-work-are-you-doing"]], "\u201cSlurm Cheat Sheet\u201d": [[17, null], [17, null]], "Walltime at the different clusters": [[17, "walltime-at-the-different-clusters"]], "Interactive jobs": [[17, "interactive-jobs"]], "Try interactive and run RStudio": [[17, "try-interactive-and-run-rstudio"]], "Type-Along": [[17, "type-along-0"], [17, "type-along-1"]], "Job scripts (batch)": [[17, "job-scripts-batch"]], "Try batch job": [[17, "try-batch-job"]], "A simple job script template": [[17, "a-simple-job-script-template"]], "Other Slurm tools": [[17, "other-slurm-tools"]], "\u201cWhy not always use an interactive session?\u201d": [[17, "exercise-0"]], "\u201cWhich approach is best in the following use cases? Batch jobs or interactive sessions?\u201d": [[17, "exercise-1"]], "\u201cAnswer\u201d": [[17, "solution-1"]], "Links": [[17, "links"]], "Exercise at home": [[17, "homework-0"]], "SNIC": [[18, "snic"]], "SNIC HPC Clusters": [[18, "snic-hpc-clusters"]], "Locally financed systems for local users": [[18, "locally-financed-systems-for-local-users"]], "SNIC Science Cloud (SSC)": [[18, "snic-science-cloud-ssc"]], "STORAGE": [[18, "storage"]], "SNIC Storage: Swestore": [[18, "snic-storage-swestore"]], "SNIC training": [[18, "snic-training"]], "Software and tools": [[19, "software-and-tools"]], "Modules": [[19, "modules"]], "Installed software": [[19, "installed-software"]], "Hands on using a tool": [[19, "exercise-0"]], "Installed databases": [[19, "installed-databases"]], "Run own scripts or programs": [[19, "run-own-scripts-or-programs"]], "Demo: Run a Fortran program": [[19, "exercise-1"]], "Specific software user guides": [[20, "specific-software-user-guides"]], "Allinea DDT": [[20, "allinea-ddt"]], "GAMESS-US": [[20, "gamess-us"]], "Garli": [[20, "garli"]], "Gaussian": [[20, "gaussian"]], "GROMACS": [[20, "gromacs"]], "Julia": [[20, "julia"]], "MATLAB": [[20, "matlab"]], "MOLCAS": [[20, "molcas"]], "NVIDIA Deep Learning Frameworks": [[20, "nvidia-deep-learning-frameworks"]], "Perl packages": [[20, "perl-packages"]], "R": [[20, "r"]], "Tensorflow": [[20, "tensorflow"]], "WRF": [[20, "wrf"]], "Getting started": [[21, "getting-started"]], "Apply for a user account": [[21, "apply-for-a-user-account"]], "Apply for a project": [[21, "apply-for-a-project"]], "Setting up two-factor authentication for UPPMAX": [[21, "setting-up-two-factor-authentication-for-uppmax"]], "Apply for CPU-time - Policy and Instructions": [[21, "apply-for-cpu-time-policy-and-instructions"]], "Disk storage guide": [[21, "disk-storage-guide"]], "Today\u2019s content": [[22, "todays-content"]], "The new documentation": [[22, "the-new-documentation"]], "Q/A": [[22, "q-a"]], "User guide": [[23, "user-guide"]], "FAQ": [[23, "faq"]], "Rackham user guide": [[23, "rackham-user-guide"]], "Bianca user guide": [[23, "bianca-user-guide"]], "Snowy user guide": [[23, "snowy-user-guide"]], "Using installed software": [[23, "using-installed-software"]], "Job submission system - Slurm": [[23, "job-submission-system-slurm"]], "Linux basics": [[23, "linux-basics"]], "General HPC": [[23, "general-hpc"]], "Life Science specific": [[23, "life-science-specific"]], "Working on UPPMAX": [[24, "working-on-uppmax"]], "$HOME dir and project dir": [[24, "home-dir-and-project-dir"]], "Quota": [[24, "quota"]], "Core hours": [[24, "core-hours"]], "What kind of work will you perform?": [[24, "what-kind-of-work-will-you-perform"]], "UPPMAX Cluster overview again!": [[24, "uppmax-cluster-overview-again"]], "Common problems": [[24, "common-problems"]], "Discussion": [[24, "discussion-0"]]}, "indexentries": {}}) \ No newline at end of file diff --git a/slurm_intro.html b/slurm_intro.html new file mode 100644 index 0000000..1647de4 --- /dev/null +++ b/slurm_intro.html @@ -0,0 +1,799 @@ + + + + + + + Using the compute nodes — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Using the compute nodes

+
+

Objectives

+
    +

  • This is a short introduction in how to reach the compute nodes

    • +

  • Wednesday afternoon is wedded to this topic!

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 13.30-14.30 (10 min break)

    • +

  • Theory

    • +

  • Hands-on

    • +
+
+
+

Attention

+ +
+
+

Note

+
    +

  • project number: naiss2024-22-49

    • +
+
+
+

The compute nodes

+

When you are logged in, you are on a login node. +There are two types of nodes:

+ + + + + + + + + + + + + + +

Type

Purpose

Login node

Start jobs for worker nodes, do easy things.

Compute nodes

Do hard calculations, either from scripts of an interactive session.

+graph TB + + Node1 -- interactive --> SubGraph2Flow + Node1 -- sbatch --> SubGraph2Flow + subgraph "Snowy" + SubGraph2Flow(calculation nodes) + end + + thinlinc -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + terminal/thinlinc -- usr --> Node1 + terminal -- usr-sensXXX + 2FA + VPN ----> SubGraph1Flow + Node1 -- usr-sensXXX + 2FA + no VPN ----> SubGraph1Flow + + subgraph "Bianca" + SubGraph1Flow(Bianca login) -- usr+passwd --> private(private cluster) + private -- interactive --> calcB(calculation nodes) + private -- sbatch --> calcB + end + + subgraph "Rackham" + Node1[Login] -- interactive --> Node2[calculation nodes] + Node1 -- sbatch --> Node2 + end +
+
+

Slurm, sbatch, the job queue

+
    +

  • Problem: 1000 users, 300 nodes, 5000 cores

    • +

  • We need a queue:

    +
      +

    • Slurm is a job scheduler

      • +
    +
    • +

  • You define jobs to be run on the compute nodes and therefore sent to the queue.

    • +
+
+

Jobs

+
    +

  • Job = what happens during booked time

    • +

  • Described in

    +
      +

    • a script file or

      • +

    • the command-line (priority over script)

      • +
    +
    • +

  • The definitions of a job:

    +
      +

    • Slurm parameters (flags)

      • +

    • Load software modules

      • +

    • (Navigate in file system)

      • +

    • Run program(s)

      • +

    • (Collect output)

      • +

    • … and more

      • +
    +
    • +
+
+

“Some keywords”

+
    +

  • A program may run serially and then needs only ONE compute thread, which will occupy 1 core, which is a physical unit of the CPU on the node.

    +
      +

    • You should most often just book 1 core. If you require more than 7 GB you can allocate more cores and you will get multiples of 7 GB.

      • +
    +
    • +

  • A program may run in parallel and then needs either several threads or several tasks, both occupying several cores.

    +
      +

    • If you need all 128 GB RAM (actually 112) or all 16 cores for your job, book a complete node.

      • +
    +
    • +
+
+
+
+

Slurm parameters

+
    +

  • 1 mandatory setting for jobs:

    +
      +

    • Which compute project? (-A)

      • +
    +
    • +

  • 3 settings you really should set:

    +
      +

    • Type of queue or partition? (-p)

      +
        +

      • core for most jobs and default!

        • +

      • node for larger jobs

        • +

      • for short development jobs and tests: devcore, devel)

        • +
      +
      • +

    • How many cores? (-n)

      +
        +

      • up to 16 for core job

        • +
      +
      • +

    • How long at most? (-t)

      • +
    +
    • +

  • If in doubt:

    +
      +

    • -p core

      • +

    • -n 1

      • +

    • -t 10-00:00:00

      • +
    +
    • +
+
+
+

The queue

+
    +

  • How does the queue work?

    • +

  • Let’s look graphically at jobs presently running.

    • +
+

Image

+
    +

  • x-axis: cores, one thread per core

    • +

  • y-axis: time +

    • +

  • We see some holes where we may fit jobs already!

    • +

  • Let’s see which type of jobs that can fit!

    • +
+

Image

+
+
    +

  • 4 one-core jobs can run immediately (or a 4-core wide job).

    +
      +

    • The jobs are too long to fit at core number 9-13.

      • +
    +
    • +
+

Image +

+
    +

  • A five-core job has to wait.

    +
      +

    • Too long to fit in cores 9-13 and too wide to fit in the last cores.

      • +
    +
    • +

  • Easiest to schedule single-threaded, short jobs

    • +
+
+

Tip

+
    +

  • You don’t see the queue graphically, however.

    • +

  • But, overall:

    +
      +

    • short and narrow jobs will start fast

      • +

    • test and development jobs can get use of specific development nodes if they are shorter than 1 hour and uses up to two nodes.

      • +

    • waste of resources unless you have a parallel program or need all the memory, e.g. 128 GB per node

      • +
    +
    • +
+
+
+
+

Core-hours

+
    +

  • Remember that you are charged CPU-hours according to booked #cores x hours

    • +

  • Example 1: 60 hours with 2 cores = 120 CPU-hours

    • +

  • Example 2: 12 hours with a full node = 192 hours

    +
      +

    • Waste of resources unless you have a parallel program using all cores or need all the memory, e.g. 128 GB per node

      • +
    +
    • +
+
+
+

Choices

+
    +

  • Work interactively with your data or develop or test

    +
      +

    • Run an Interactive session

      • +

    • $ interactive <flags> ...

      • +
    +
    • +

  • If you don’t need any live interaction with your workflow/analysis/simulation

    +
      +

    • Send your job to the slurm job batch (sbatch)

      • +

    • $ sbatch <flags> <program> or

      • +

    • $ sbatch <job script>

      • +
    +
    • +
+flowchart TD + UPPMAX(What to run on which node?) + operation_type{What type of operation/calculation?} + interaction_type{What type of interaction?} + login_node(Work on login node) + interactive_node(Work on interactive node) + calculation_node(Schedule for calculation node) + + UPPMAX-->operation_type + operation_type-->|light,short|login_node + operation_type-->|heavy,long|interaction_type + interaction_type-->|Direct|interactive_node + interaction_type-->|Indirect|calculation_node +
+
+

What kind of compute work are you doing?

+
    +

  • Compute bound

    +
      +

    • you use mainly CPU power

      • +

    • does the software support threads or MPI?

      +
        +

      • Threads/openMP are rather often supported. Use several cores!

        • +

      • MPI (Message Passing Interface) allows for inter-node jobs but are seldom supported for bioinformatics software. You could use several nodes!

        • +
      +
      • +
    +
    • +

  • Memory bound

    +
      +

    • if the bottlenecks are allocating memory, copying/duplicating

      • +

    • use more cores up to 1 node, perhaps using a “fat” node.

      • +
    +
    • +
+
+

“Slurm Cheat Sheet”

+
    +

  • -A project number

    • +

  • -t wall time

    • +

  • -n number of cores

    • +

  • -N number of nodes (can only be used if your code is parallelized with MPI)

    • +

  • -p partition

    +
      +

    • core is default and works for jobs narrower than 16 cores

      • +

    • node can be used if you need the whole node and its memory

      • +
    +
    • +
+
+
+
+

Walltime at the different clusters

+
    +

  • Rackham: 10 days

    • +

  • Snowy: 30 days

    • +

  • Bianca: 10 days

    • +
+
+
+
+

Interactive jobs

+
    +

  • Most work is most effective as submitted jobs, but e.g. development needs responsiveness

    • +

  • Interactive jobs are high-priority but limited in -n and -t

    • +

  • Quickly give you a job and logs you in to the compute node

    • +

  • Require same Slurm parameters as other jobs

    • +

  • Log in to compute node

    +
      +

    • $ interactive ...

      • +
    +
    • +

  • Logout with <Ctrl>-D or logout

    • +

  • To use an interactive node, in a terminal, type:

    • +
+
interactive -A [project name] -p core -n [number_of_cores] -t [session_duration]
+
+
+

For example:

+
interactive -A naiss2024-22-49 -p core -n 2 -t 8:0:0
+
+
+

This starts an interactive session using project naiss2024-22-49 +that uses 2 cores and has a maximum duration of 8 hours.

+
+

Try interactive and run RStudio

+
    +

  • We recommend using at least two cores for RStudio, and to get those resources, you must start an interactive job.

    • +
+
+

Type-Along

+
    +

  • Use ThinLinc

    • +

  • Start interactive session on compute node (2 cores)

    • +

  • If you already have an interactive session going on use that.

    +
      +

    • If you don’t find it, do

      • +
    +

    $ squeue

    +
      +

    • find your session, ssh to it, like:

      • +
    +

    $ ssh r483

    +
    • +

  • If you have no ongoing session:

    • +
+

$ interactive -A naiss2024-22-49 -p devcore -n 2 -t 60:00

+
    +

  • Once the interactive job has begun you need to load needed modules, even if you had loaded them before in the login node

    • +

  • You can check which node you are on?

    +

    $ hostname

    +
    • +

  • Also try:

    +

    $ srun hostname

    +
      +

    • This will give several output lines resembling the number of cores you allocated.

      • +

    • How many in this case??

      • +
    +

    [bjornc@r483 ~]$

    +
    • +

  • Load an RStudio module and an R_packages module (if not loading R you will have to stick with R/3.6.0) and run “rstudio” from there.

    +

    $ ml R_packages/4.2.1

    +

    $ ml RStudio/2022.07.1-554

    +
    • +

  • Start rstudio, keeping terminal active (&)

    +

    $ rstudio &

    +
    • +

  • Slow to start?

    • +

  • Depends on:

    +
      +

    • number of packages

      • +

    • if you save a lot of data in your RStudio workspace, to be read during start up.

      • +
    +
    • +

  • Quit RStudio!

    • +

  • Log out from interactive session with <Ctrl>-D or logout or exit

    • +
+
+
+
+
+

Job scripts (batch)

+
    +

  • Batch scripts can be written in any scripting language. We will use BASH

    • +

  • Make first line be #!/bin/bash in the top line

    +
      +

    • It is good practice to end the line with -l to reload a fresh environment with no modules loaded.

      • +

    • This makes you sure that you don’t enable other software or versions that may interfere with what you want to do in the job.

      • +
    +
    • +

  • Before the job content, add the batch flags starting the lines with the keyword #SBATCH, like:

    +
      +

    • #SBATCH -t 2:00:00

      • +

    • #SBATCH -p core

      • +

    • #SBATCH -n 3

      • +
    +
    • +

  • # will be ignored by bash and can run as an ordinary bash script

    • +

  • if running the script with the command sbatch <script> the #SBATCH lines will be interpreted as slurm flags

    • +
+
+

Try batch job

+
+

Type-Along

+
    +

  • Write a bash script called jobscript.sh

    +
      +

    • You can be in your ~ folder

      • +
    +
    • +

  • To make it more efficient Copy-paste the code below.

    • +
+
+
+

A simple job script template

+
#!/bin/bash
+
+#SBATCH -A naiss2024-22-49  # Project ID
+
+#SBATCH -p devcore  # Asking for cores (for test jobs and as opposed to multiple nodes) 
+
+#SBATCH -n 1  # Number of cores
+
+#SBATCH -t 00:10:00  # Ten minutes
+
+#SBATCH -J Template_script  # Name of the job
+
+# go to some directory
+
+cd /proj/naiss2024-22-49/
+pwd -P
+
+# load software modules
+
+module load bioinfo-tools
+module list
+
+# do something
+
+echo Hello world!  
+
+
+
    +

  • Run it:

    +

    $ sbatch jobscript.sh

    +
    • +
+
+

Tip

+

Do you need more resources?

+

Do you need more memory than 128 GB or GPU:s?

+
    +

  • -C mem256GB allocate a fat node with 256 GB RAM

    • +

  • -C mem512GB allocate a fat node with 512 GB RAM

    • +

  • -C gpu

    • +

  • -p node must be used when allocating these nodes

    • +

  • GPU example asking for a node, using one GPU and 3 CPU cores. TIP: ask always for more than one hour!

    +
      +

    • interactive -A <proj> -n 3 -C gpu --gres=gpu:1 -t 01:10:00

      • +
    +
    • +
+
+
+

Note

+

Some Limits

+
    +

  • There is a job wall time limit of ten days (240 hours).

    • +

  • We restrict each user to at most 5000 running and waiting jobs in total.

    • +

  • Each project has a 30 days running allocation of CPU hours.

    +
      +

    • We do not forbid running jobs after the allocation is over-drafted

      • +

    • Instead allow to submit jobs with a very low queue priority, so that you may be able to run your jobs anyway, if a sufficient number of nodes happens to be free on the system.

      • +
    +
    • +
+
+
+
+
+
+

Other Slurm tools

+
    +

  • squeue — quick info about jobs in queue

    • +

  • jobinfo — detailed info about jobs

    • +

  • finishedjobinfo — summary of finished jobs

    • +

  • jobstats — efficiency of booked resources

    +
      +

    • use eog to watch the png output files

      • +
    +
    • +

  • bianca_combined_jobinfo

    • +
+
+

“Slurm Cheat Sheet”

+
    +

  • -A project number

    • +

  • -t wall time

    • +

  • -n number of cores

    • +

  • -N number of nodes (can only be used if your code is parallelized with MPI)

    • +

  • -p partition

    +
      +

    • core is default and works for jobs narrower than 16 cores

      • +

    • node can be used if you need the whole node and its memory

      +
        +

      • must be used when allocating the fat nodes, see below

        • +
      +
      • +
    +
    • +

  • -C mem256GB allocate a fat node with 256 GB RAM

    • +

  • -C mem512GB allocate a fat node with 512 GB RAM

    • +

  • -C gpu

    • +
+

Batch jobs

+
    +

  • Two alternatives

    +
      +

    • sbatch <jobscript with all #SBATCH options>

      • +

    • sbatch <options that will be prioritized over the options within the jobs script> <jobscript>

      +
        +

      • can for instance be used if you just want to test with, for instance, fewer cores and shorter time

        • +

      • Example: sbatch -t  60:00 -p devcore -n 2 job.sh

        • +
      +
      • +
    +
    • +
+

Interactive

+
    +

  • interactive -A <project> <other options if not using default settings>

    • +

  • load your modules when session starts

    • +
+
+
+
+

Exercises

+
+

“Why not always use an interactive session?”

+ +
+
+

“Which approach is best in the following use cases? Batch jobs or interactive sessions?”

+
    +

  1. Long jobs

    2. +

  2. Short jobs with interactive “run-time”/interactive user input

    3. +

  3. Short jobs without interactive “run-time”/interactive user input

    4. +

  4. Test/debugging/developing code

    5. +

  5. Playing with and plotting large data

    6. +
+
+ +
+ +
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/snic.html b/snic.html new file mode 100644 index 0000000..c49c818 --- /dev/null +++ b/snic.html @@ -0,0 +1,263 @@ + + + + + + + SNIC — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

SNIC

+
+

SNIC HPC Clusters

+

https://www.snic.se/resources/compute-resources/

+
    +

  • Mostly Intel-based CPU:s and Nvidia GPU:s

    • +

  • UPPMAX:

    +
      +

    • Rackham - General computational resource and also focused towards Life Science

      • +

    • Bianca - SNIC SENS, resource dedicated to research on sensitive data

      • +

    • Miarka

      • +
    +
    • +

  • PDC (KTH):

    +
      +

    • Dardel - General computational resource

      • +

    • Not intel/Nvidia but Cray/AMD CPU:s and GPU:s

      • +
    +
    • +

  • NSC, Linköping:

    +
      +

    • Tetralith - General computational resource

      • +
    +
    • +

  • HPC2N, Umeå:

    +
      +

    • Kebnekaise - General computational resource

      • +
    +
    • +

  • C3SE, Chalmers:

    +
      +

    • Alvis - Accelerator-based resource dedicated to research using AI techniques +AND the international

      • +
    +
    • +

  • LUMI, Finland

    +
      +

    • LUMI - General computational resource, aimed at larger projects in Sweden

      • +

    • Not intel/Nvidia but Cray/AMD CPU:s and GPU:s

      • +
    +
    • +
+
+

Locally financed systems for local users

+
    +

  • UPPMAX

    +
      +

    • Snowy - GPU:s and long runs

      • +
    +
    • +

  • LUNARC, Lund

    +
      +

    • Aurora

      • +
    +
    • +

  • C3SE, Chalmers

    +
      +

    • Vera

      • +
    +
    • +

  • NSC, Linköping:

    +
      +

    • BerzeLiUs - for AI and machine learning

      • +
    +
    • +
+
+
+
+

SNIC Science Cloud (SSC)

+
    +

  • Complement, offering “advanced functionality” to users who:

    +
      +

    • need more flexible access to resources

      +
        +

      • e.g. more control over the OS:s and software environments

        • +
      +
      • +

    • want to develop software as a service

      • +

    • want to explore recent technology such as for

      +
        +

      • IoT applications

        • +

      • “Big Data” e.g. Apache Hadoop/Spark

        • +
      +
      • +
    +
    • +

  • The UPPMAX cloud Dis (Swedish word for “haze”) was introduced in October 2017 and upgraded during 2020.

    + +
    • +

  • Other clouds

    +
      +

    • Umeå University (North, HPC2N).

      • +

    • Chalmers University (West, C3SE).

      • +
    +
    • +
+
+
+

STORAGE

+
+

SNIC Storage: Swestore

+
    +

  • The purpose of Swestore allocations, granted by Swedish National Allocations Committee (SNAC), is to provide large scale data storage for “live” or “working” research data, also known as active research data.

    • +

  • https://www.snic.se/resources/swestore/

    • +
+
+
+
+

SNIC training

+

https://www.snic.se/support/snic-training/

+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/software.html b/software.html new file mode 100644 index 0000000..86d011a --- /dev/null +++ b/software.html @@ -0,0 +1,285 @@ + + + + + + + Software and tools — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Software and tools

+
+

Objectives

+
    +

  • We’ll briefly get overviews over

    +
      +

    • software tools on UPPMAX

      • +

    • databases

      • +
    +
    • +

  • Introduction guide for installing own software or packages

    • +

  • Very short introduction to developing own programs

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 13:00-13:30

    • +

  • Theory and type-along

    • +
+
+
    +

  • 800+ programs and packages are installed.

    • +

  • To avoid chaos and collisions, they are managed by a module system.

    • +

  • This system keeps installed software hidden by default, and users have to explicitly tell their terminal which version of which software they need.

    • +

  • The modules are most often available across cluster (except for Miarka)

    • +
+
+

Note

+
    +

  • Bioinformatics tools require loading the “bioinfo-tools” module first.

    • +
+
+
+

Modules

+ +
+
+

Installed software

+ +
+

Hands on using a tool

+
    +

  1. use matlab

    2. +
+
$ matlab &
+
+
+
    +

  • Does not work!

    • +

  • Load module first

    • +
+
$ module avail matlab
+
+
+
    +

  • module load matlab will start Default version (often latest) demarked with a D in the list

    • +

  • Let’s load a specific version, often good for reproducibility.

    • +
+
$ module load matlab/R2020b
+
+$ matlab &
+
+
+
    +

  • Matlab starts (if X11 is active)

    • +
+
    +

  1. use Samtools

    2. +
+
$ module load samtools
+
+        "These module(s) or extension(s) exist but cannot be loaded as requested: "samtools""
+
+
+
$ module load bioinfo-tools samtools
+
+
+
    +

  • Bioinformatic tools are hidden by default

    • +
+
+
+
+

Installed databases

+ +
+
+

Run own scripts or programs

+
    +

  • Unless your script or program is in the active path, you run it by the full path or ./<file> if you are in the present directory.

    • +
+
+

Demo: Run a Fortran program

+
    +

  • Run the program “sunray” located in: /proj/introtouppmax/labs/sunray

    • +
+
+ +
+

Keypoints

+
    +

  • Centrally installed software are reached through the module system and available throughout all nodes.

    • +

  • Your own installed software, scripts, python packages etcetera are available from their paths.

    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/specific.html b/specific.html new file mode 100644 index 0000000..324e69a --- /dev/null +++ b/specific.html @@ -0,0 +1,212 @@ + + + + + + + Specific software user guides — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Specific software user guides

+
+

Allinea DDT

+

DDT

+
+
+

GAMESS-US

+

GAMESS-US

+
+
+

Garli

+

Garli

+
+
+

Gaussian

+

Gaussian

+
+
+

GROMACS

+

GROMACS

+
+
+

Julia

+

Julia

+
+
+

MATLAB

+

MATLAB

+
+
+

MOLCAS

+

MOLCAS

+
+
+

NVIDIA Deep Learning Frameworks

+

NVIDIA Deep Learning Frameworks

+
+
+

Perl packages

+ +
+
+

Python

+

Python

+
+
+

R

+ +
+
+

Tensorflow

+

Tensorflow

+
+
+

WRF

+

WRF

+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/start.html b/start.html new file mode 100644 index 0000000..51fa53d --- /dev/null +++ b/start.html @@ -0,0 +1,173 @@ + + + + + + + Getting started — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Getting started

+

In order to use UPPMAX resources, you need an UPPMAX user account and a project. Your user account is a personal log-in to our systems. Computer resources like CPU-hours and disk storage are allocated to projects.

+

UPPMAX is also a cluster and cloud provider for Uppsala University educational courses. See the university course information page if you are interested in teaching with us.

+

If the results you get from using the resources at UPPMAX end up in a paper or are presented at a workshop or conference, we kindly ask you to acknowledge UPPMAX and SNIC.

+
+

Apply for a user account

+

Apply for a user account

+
+
+

Apply for a project

+

Apply for a project

+
+
+

Setting up two-factor authentication for UPPMAX

+

two-factor authentication

+
+
+

Apply for CPU-time - Policy and Instructions

+

Apply for CPU-time

+
+
+

Disk storage guide

+

Disk storage guide

+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/summary.html b/summary.html new file mode 100644 index 0000000..ba38bc4 --- /dev/null +++ b/summary.html @@ -0,0 +1,240 @@ + + + + + + + Summary — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Summary

+
+

Today’s content

+
    +

  • Overview

    +
      +

    • NAISS makes available large-scale high-performance computing resources, storage capacity, and advanced user support, for Swedish research.

      • +

    • UPPMAX runs the local resources placed at Uppsala Universtiy

      • +

    • A cluster consists of several inter-connected computers that can work individually or together.

      • +
    +
    • +

  • Log in

    +
      +

    • Terminals

      • +

    • ThinLinc

      • +
    +
    • +

  • Linux introduction

    +
      +

    • Understanding

      • +

    • Basic commands

      • +
    +
    • +

  • UPPMAX & support

    • +

  • UPPMAX clusters

    +
      +

    • Rackham - general

      • +

    • Snowy - local compute, and courses

      • +

    • Bianca - sensitive data

      • +
    +
    • +

  • NAISS

    +
      +

    • projects

      • +
    +
    • +

  • Software and tools

    +
      +

    • software modules

      • +

    • installing

      • +
    +
    • +

  • Using the compute nodes

    +
      +

    • queues

      • +

    • interactive

      • +

    • batch

      • +
    +
    • +

  • Working on UPPMAX

    +
      +

    • some tips

      • +
    +
    • +

  • NAISS Sens and Bianca

    +
      +

    • No direct internet connection

      • +

    • Wharf

      • +
    +
    • +
+
+
+

The new documentation

+ +
+

Tip

+ +
+
+
+

Q/A

+
    +

  • Do we need break-out rooms?

    • +
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/userguide.html b/userguide.html new file mode 100644 index 0000000..36cf4ca --- /dev/null +++ b/userguide.html @@ -0,0 +1,255 @@ + + + + + + + User guide — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

User guide

+
+

FAQ

+

FAQ

+
+
+

What is a cluster?

+
    +

  • A network of computers, each computer working as a node.

    • +

  • From small scale RaspberryPi cluster :-)

    • +

  • To supercomputers like Rackham.

    • +

  • Each node contains several processor cores and RAM and a local disk called scratch.

    • +

  • The user logs in to login nodes via Internet through ssh or Thinlinc.

    +
      +

    • Here the file management and lighter data analysis can be performed.

      • +
    +
    • +

  • The calculation nodes must be used for intense computing.

    +
      +

    • “Normal” softwares use one core.

      • +

    • Parallelized software can utilize several cores or even several nodes. Keywords signalizing this are e.g.:

      • +
    • "multi-threaded", "openMP", "shared memory", "MPI", "distributed memory".
+
      +
      +
      • +

    • To let your software run on the calculation nodes

      • +

    • start an “interactive session” or

      • +

    • “submit a batch job”.

      • +
    +
    • +

  • All nodes can access:

    +
      +

    • Your home directory on Domus or Castor

      • +

    • Your project directories on Crex or Castor

      • +

    • Its own local scratch disk (2-3 TB)

      • +
    +
    • +
+
+
+

Rackham user guide

+ +
+
+

Bianca user guide

+ +
+
+

Snowy user guide

+ +
+
+

Using installed software

+ +
+
+

Job submission system - Slurm

+ +
+
+

Linux basics

+ +
+
+

General HPC

+ +
+
+

Life Science specific

+
    +

  • Applying for a Life Science project

    • +

  • Running LifeScope on UPPMAX

    • +

  • Starting Integrative Genomics Viewer (IGV) on Rackham/Snowy

    • +

  • Test Galaxy on UPPMAX

    • +

  • Using CRAM to compress BAM files

    • +

  • Accessing Swegen

    • +
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/working.html b/working.html new file mode 100644 index 0000000..bceee92 --- /dev/null +++ b/working.html @@ -0,0 +1,366 @@ + + + + + + + Working on UPPMAX — UPPMAX intro course — part 1 0.1 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +
+

Working on UPPMAX

+
+

Objectives

+
    +

  • Let’s get some tips and best practices

    • +
+
+
+

Instructor note

+
    +

  • Approx timing: 14.30-14:45

    +
      +

    • May be expanded if good discussions and interactions

      • +

    • Helpers and users can share their solutions

      • +
    +
    • +

  • Information

    • +

  • discussion

    • +
+
+
+

$HOME dir and project dir

+
+

Quota

+
    +

  • Disk usage and number of files

    • +

  • $HOME has always 32 GB and 300,000 files

    • +

  • You typically have one project per project, their size dependent on type of project

    • +

  • Compute projects both have disk space (128 GB) and computing time attached to them

    • +

  • Check your quota with

    • +
+
$ uquota
+
+
+
+
+

Core hours

+
    +

  • You get core hours only from compute project

    • +

  • When they are used you can still get “BONUS” jobs if the resources allow.

    • +

  • You can find your current projects (and other projects that you have run jobs in) with the program projinfo.

    • +
+
$ projinfo [<username>]
+
+
+

or

+
$ projinfo [<project name>]
+
+
+
    +

  • For example, if your project is named NAISS 2017/1-334 you specify naiss2017-1-334

    • +
+
+
+
+

What kind of work will you perform?

+

+@startuml +start + +:login; +:Rackham login Node; +if (Do CPU/memory intensive work) then (yes) + :Use calculation node; + if (Do interactive work) then (yes) + :$ interactive -A <proj> <options>; + else (no) + :Make a batch script; + :$ sbatch <script>; + endif +else (no) + :Stay on login node and load your module(s): + $ module load <software/tool>; + :Run tool: + $ <toolname> [- options, input, output]; +endif +:finish; +stop +@enduml +

+
+

UPPMAX Cluster overview again!

+graph TB + + Node1 -- interactive --> SubGraph2Flow + Node1 -- sbatch --> SubGraph2Flow + subgraph "Snowy" + SubGraph2Flow(calculation nodes) + end + + thinlinc -- usr-sensXXX + 2FA----> SubGraph1Flow + Node1 -- usr-sensXXX + 2FA----> SubGraph1Flow + subgraph "Bianca" + SubGraph1Flow(Bianca login) -- usr+passwd --> private(private cluster) + private -- interactive --> calcB(calculation nodes) + private -- sbatch --> calcB + end + + subgraph "Rackham" + Node1[Login] -- interactive --> Node2[calculation nodes] + Node1 -- sbatch --> Node2 + end +
+
+
+

Common problems

+
    +

  • Conda environment clashes with loaded python modules

    • +

  • Forgotten environment variables defined in your .bashrc may give unexeptected errors when you run other programs or new versions of a program

    • +

  • A full $HOME folder may cause unexpected errors

    +
      +

    • check with uquota

      • +
    +
    • +
+
+

Tip

+

.bashrc

+
    +

  • Do not fill ~/.bashrc with too much

    +
      +

    • Avoid loading modules, unless you are totally sure that those will not interact with other things you are doing

      • +

    • Typical problem: loading several conflicting conda environments and python modules

      • +
    +
    • +

  • Good place to have:

    +
      +

    • aliases

      • +

    • source files (if needed)

      +
        +

      • these can set some environment variables

        • +
      +
      • +
    +
    • +

  • Example from Björn C

    • +
+
alias lt='ls -lrt'
+alias sq='squeue -u $USER'
+alias sc='scancel'
+alias rm='rm -i'
+alias less='less -R'
+
+
+
    +

  • When done, start a new bash session in a new terminal or type bash in the present one.

    • +
+

Conda

+
    +

  • You cannot mix Conda environments with python software module

    • +

  • Conda is very nice for Bianca, because of the ease of install packages without the internet connection.

    • +

  • For Rackham we suggest Python virtual environemnts with virtualenv, see Python course

    +
      +

    • Unless the conda environment is straight-forward to install and is well-defined.

      • +
    +
    • +
+

Things worked before but do not now

+
    +

  • Check uquota !

    • +

  • Very often a full disk space is the reason

    • +
+
+
+

Discussion

+
    +

  • Any input from other teachers!

    • +
+
+
+

Keypoints

+
    +

  • Use your disk spaces wisely

    +
      +

    • home folder just for general stuff and files needed by several projects

      +
        +

      • always read and write protected for others by default

        • +
      +
      • +

    • otherside project folder which will more easily become public for others

      +
        +

      • by default available for all project members.

        • +
      +
      • +
    +
    • +

  • Use the computing resources wisely

    +
      +

    • low intensity work on login node

      • +

    • high intensity work on compute nodes (core hours are counted)

      +
        +

      • for development use the interactive sessions

        • +

      • otherwise make batch jobs!

        • +
      +
      • +
    +
    • +
+
+
+
+ + +
+
+ +
+
+
+
+ + + + \ No newline at end of file