Skip to content

Commit

Permalink
even more redaction
Browse files Browse the repository at this point in the history
  • Loading branch information
cmerwich committed Apr 21, 2020
1 parent f5b82f8 commit 317988a
Show file tree
Hide file tree
Showing 12 changed files with 905 additions and 529 deletions.
73 changes: 39 additions & 34 deletions annotation/2.annotation_aid.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 18,
"metadata": {},
"outputs": [
{
Expand All @@ -95,7 +95,7 @@
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n",
"Notebook last updated by Christiaan at 2020-04-15 22:23:58.205031\n"
"Notebook last updated by Christiaan at 2020-04-21 18:36:01.605605\n"
]
}
],
Expand Down Expand Up @@ -128,14 +128,16 @@
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"execution_count": 25,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"rate limit is 60 requests per hour, with 0 left for this hour\n"
"rate limit is 60 requests per hour, with 60 left for this hour\n"
]
},
{
Expand All @@ -149,12 +151,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\tconnecting to online GitHub repo annotation/app-bhsa ... failed\n",
"GitHub says: 403 {\"message\": \"API rate limit exceeded for 85.203.44.68. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)\", \"documentation_url\": \"https://developer.github.com/v3/#rate-limiting\"}\n",
"The offline TF-app may not be the latest\n",
"\tconnecting to online GitHub repo annotation/app-bhsa ... connected\n",
"Using TF-app in /Users/Christiaan/text-fabric-data/annotation/app-bhsa/code:\n",
"\trv1.3=#f38d56bd757e87fe12d0c125e1ca52ee4376127b (latest? release)\n",
"rate limit is 60 requests per hour, with 0 left for this hour\n"
"\trv1.3=#f38d56bd757e87fe12d0c125e1ca52ee4376127b (latest release)\n",
"rate limit is 60 requests per hour, with 55 left for this hour\n"
]
},
{
Expand All @@ -168,12 +168,13 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\tconnecting to online GitHub repo etcbc/bhsa ... failed\n",
"GitHub says: 403 {\"message\": \"API rate limit exceeded for 85.203.44.68. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)\", \"documentation_url\": \"https://developer.github.com/v3/#rate-limiting\"}\n",
"The offline data may not be the latest\n",
"\tconnecting to online GitHub repo etcbc/bhsa ... connected\n",
"\tdownloading https://github.com/ETCBC/bhsa/releases/download/v1.6/tf-2017.zip ... \n",
"\tunzipping ... \n",
"\tsaving data\n",
"Using data in /Users/Christiaan/text-fabric-data/etcbc/bhsa/tf/2017:\n",
"\trv1.6=#bac4a9f5a2bbdede96ba6caea45e762fe88f88c5 (latest? release)\n",
"rate limit is 60 requests per hour, with 0 left for this hour\n"
"\trv1.6=#bac4a9f5a2bbdede96ba6caea45e762fe88f88c5 (latest release)\n",
"rate limit is 60 requests per hour, with 46 left for this hour\n"
]
},
{
Expand All @@ -187,12 +188,13 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\tconnecting to online GitHub repo etcbc/phono ... failed\n",
"GitHub says: 403 {\"message\": \"API rate limit exceeded for 85.203.44.68. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)\", \"documentation_url\": \"https://developer.github.com/v3/#rate-limiting\"}\n",
"The offline data may not be the latest\n",
"\tconnecting to online GitHub repo etcbc/phono ... connected\n",
"\tdownloading https://github.com/ETCBC/phono/releases/download/1.2/tf-2017.zip ... \n",
"\tunzipping ... \n",
"\tsaving data\n",
"Using data in /Users/Christiaan/text-fabric-data/etcbc/phono/tf/2017:\n",
"\tr1.2 (latest? release)\n",
"rate limit is 60 requests per hour, with 0 left for this hour\n"
"\tr1.2=#1ac68e976ee4a7f23eb6bb4c6f401a033d0ec169 (latest release)\n",
"rate limit is 60 requests per hour, with 37 left for this hour\n"
]
},
{
Expand All @@ -206,12 +208,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\tconnecting to online GitHub repo etcbc/parallels ... failed\n",
"GitHub says: 403 {\"message\": \"API rate limit exceeded for 85.203.44.68. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)\", \"documentation_url\": \"https://developer.github.com/v3/#rate-limiting\"}\n",
"The offline data may not be the latest\n",
"\tconnecting to online GitHub repo etcbc/parallels ... connected\n",
"Using data in /Users/Christiaan/text-fabric-data/etcbc/parallels/tf/2017:\n",
"\tr1.2 (latest? release)\n",
"rate limit is 60 requests per hour, with 0 left for this hour\n"
"\tr1.2 (latest release)\n",
"rate limit is 60 requests per hour, with 32 left for this hour\n"
]
},
{
Expand All @@ -225,11 +225,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\tconnecting to online GitHub repo cmerwich/bh-reference-system ... failed\n",
"GitHub says: 403 {\"message\": \"API rate limit exceeded for 85.203.44.68. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)\", \"documentation_url\": \"https://developer.github.com/v3/#rate-limiting\"}\n",
"The offline data may not be the latest\n",
"\tconnecting to online GitHub repo cmerwich/bh-reference-system ... connected\n",
"Using data in /Users/Christiaan/text-fabric-data/cmerwich/bh-reference-system/tf/2017:\n",
"\trv1.0 (latest? release)\n",
"\trv1.0 (latest release)\n",
" | 0.00s No structure info in otext, the structure part of the T-API cannot be used\n"
]
},
Expand Down Expand Up @@ -559,7 +557,7 @@
" mod=(\n",
" 'cmerwich/bh-reference-system/tf'\n",
" ),\n",
" hoist=globals(),\n",
" hoist=globals()\n",
")"
]
},
Expand Down Expand Up @@ -9569,7 +9567,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -9644,7 +9642,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -9662,7 +9660,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 28,
"metadata": {
"scrolled": false
},
Expand Down Expand Up @@ -9690,7 +9688,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 29,
"metadata": {
"scrolled": false
},
Expand Down Expand Up @@ -12688,6 +12686,13 @@
"source": [
"show_text(results, highlights)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
20 changes: 10 additions & 10 deletions confrontation/Psalms_075.ann
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,13 @@ T62 Mention 476 489 KL QRNJ RC<JM
T63 Mention 490 494 >GD<
T64 Mention 495 502 TRWMMNH
T65 Mention 503 513 QRNWT YDJQ
* Coreference +K >LHJM +K +K
* Coreference HWDJNW HWDJNW
* Coreference SPRW NPL>WTJ
* Coreference >NJ >CPV >NKJ TKNTJ
* Coreference NMGJM >RY W-KL JCBJ
* Coreference TRJMW TRJMW TDBRW
* Coreference JCPJL JRJM
* Coreference JJN XMR
* Coreference >NJ >GJD >ZMRH >GD<
* Coreference TRWMMNH QRNWT YDJQ
* Coreference T7 T8 T11 T14
* Coreference T6 T9
* Coreference T12 T13
* Coreference T17 T18 T22 T23
* Coreference T19 T20
* Coreference T30 T32 T36
* Coreference T44 T46
* Coreference T49 T50
* Coreference T57 T58 T60 T63
* Coreference T64 T65
4 changes: 2 additions & 2 deletions confrontation/analyse.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
A = use(
'bhsa', version='2017',
mod=(
'cmerwich/participant-analysis/coreference/tf:clone,'
'etcbc/bh-reference-system/tf:clone'
'cmerwich/participant-analysis/coreference/tf,'
'cmerwich/bh-reference-system/tf'
),
hoist=globals(),
silent=True)
Expand Down
100 changes: 100 additions & 0 deletions confrontation/annotation_aid.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import re, collections
from IPython.display import display, HTML, Markdown

from tf.applib.repo import checkoutRepo
from tf.app import use
from tf.fabric import Fabric

def do(task):
result = task
md = f'''commit | release | local | base | subdir
--- | --- | --- | --- | ---
`{task[0]}` | `{task[1]}` | `{task[2]}` | `{task[3]}` | `{task[4]}`
'''
display(Markdown(md))

do(
checkoutRepo(org='cmerwich', repo='bh-reference-system', folder='tf', version='2017', checkout='')
)


A = use(
'bhsa', version='2017',
mod=(
'cmerwich/bh-reference-system/tf'
),
hoist=globals(),
silent=True
)

def compute_text(my_book_name, from_chapter, to_chapter):

results = []
highlights = {}

my_chapters = set(range(from_chapter, to_chapter+1))

for book in F.otype.s('book'):
book_name = T.bookName(book)

for chn in L.d(book, 'chapter'):
chapter = F.chapter.v(chn)
tup = (chn,)
if (
(my_book_name and book_name not in my_book_name)
or
(my_chapters and chapter not in my_chapters)
):
continue
for phrase in L.d(chn, 'phrase'):
typ = F.typ.v(phrase)
if typ == 'NP':
tup = tup + (phrase,)
highlights[phrase] = 'skyblue'

for phr_atom in L.d(chn, 'phrase_atom'):
if F.rela.v(phr_atom) == 'Appo':
tup = tup + (phr_atom,)
highlights[phr_atom] = 'yellow'

for w in L.d(chn, 'word'):
pdp = F.pdp.v(w)
pgn_prps = F.pgn_prps.v(w)
pgn_prde = F.pgn_prde.v(w)
pgn_verb = F.pgn_verb.v(w)
pgn_prs = F.pgn_prs.v(w)

if pdp == 'verb':
tup = tup + (w,)
highlights[w] = 'springgreen'

if pdp == 'subs':
tup = tup + (w,)
highlights[w] = 'skyblue'

if pdp == 'art':
tup = tup + (w,)
highlights[w] = 'skyblue'

if pdp == 'nmpr':
tup = tup + (w,)
highlights[w] = 'tomato'

if pdp == 'prps':
tup = tup + (w,)
highlights[w] = 'palegoldenrod'

if pdp == 'prde':
tup = tup + (w,)
highlights[w] = 'royalblue'

if pdp == 'prep' and pgn_prs in prs_set:
tup = tup + (w,)
highlights[w] = 'DarkGoldenrod'

results.append(tup)
return (results, highlights)

def show_text(results, highlights):
A.displaySetup(withNodes=True, extraFeatures='pgn_prps pgn_prde pgn_verb pgn_prs pdp typ rela ls function det st lex nametype gn nu') #ps gn nu
A.show(results, condensed=False, highlights=highlights)
Loading

0 comments on commit 317988a

Please sign in to comment.