Commit 9e15786a authored by dirk.wintergruen's avatar dirk.wintergruen
Browse files

more documentation

parent 93e9270f
......@@ -2,7 +2,6 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.7 (anaconda-default)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="R User Library" level="project" />
<orderEntry type="library" name="R Skeletons" level="application" />
......
......@@ -1111,11 +1111,10 @@ def createYearNetworks(ng,startyear,endyear,typ=None,
ynw={}
ng_an = ng.copy()
#check if progressbar is set
if progressBar is not None:
progressBar.max=endyear
progressBar.min=startyear
try: # try a different method in addition
progressBar.max_value=endyear
progressBar.min_value=startyear
......@@ -1126,7 +1125,7 @@ def createYearNetworks(ng,startyear,endyear,typ=None,
pass
iter = range(startyear,endyear)
#check if tqdm is set
if tqdm is None and progressBar is None:
from tqdm import tqdm
......@@ -1144,7 +1143,7 @@ def createYearNetworks(ng,startyear,endyear,typ=None,
nw = ng.__class__(directed=True)
old_ids = {}
logger.debug("new nw type: %s"%type(nw))
## add additiona attributes
## add additional attributes
for k, v in ng.__dict__.items():
if v:
if hasattr(v,"copy"):
......
......@@ -232,7 +232,7 @@ settings = {"pers_pers_by_membership": {
"ident_attr":"label",
"doi_all":"doi:10.5072/FK2/GPFUWZ",
"graph_all_file":"kom_pers_directed.graphml"
},
},
"pers_pers_by_deals_with_and_members": {
"description":"Personen verbunden mit Personen über die verhandelt wird und Mitglieder",
"doi":"doi:10.5072/FK2/2SVCY5",
......@@ -241,30 +241,16 @@ settings = {"pers_pers_by_membership": {
"ident_attr":"label",
"doi_all":"doi:10.5072/FK2/GPFUWZ",
"graph_all_file":"all_persons_members_and_deals_blocks.graphml"
}
}
}
if __name__ == "__main__":
import igraph
from igraph import igraphx
"""
example:
from network_extensions import igraphx
from dataverseTools import handleDataverse
dv = handleDataverse.DataverseHandler("datastore-dev.mpiwg-berlin.mpg.de",
"c4cacadd-d3ac-45f0-94b9-98d5396e302b",
"gmpg-astronomy-gremien")
current_setting = settings["pers_pers_by_membership"]
content = dv.getContent(current_setting["doi_all"],current_setting["graph_all_file"])
with open("tmp.graphml","wb") as outf:
outf.write(content.read())
gr_all = igraph.load(open("tmp.graphml","rb"))
dv_gephi_url="http://localhost:8080/workspace1"
ctn = GephiConnection(url=dv_gephi_url)
stream = GephiGraphStreamer()
......@@ -295,5 +281,7 @@ if __name__ == "__main__":
print("url error")
print(v)
"""
\ No newline at end of file
......@@ -11,7 +11,7 @@ import pickle
import pandas
import plotly.graph_objs as go
from plotly.offline import iplot
from tqdm import tqdm
from tqdm import tqdm, tqdm_notebook
from network_extensions import igraphx
from network_extensions.igraphx.graph_plotly import createFigGraph
......@@ -37,8 +37,16 @@ class MultiLayerGraph(OrderedDict):
"""
def __init__(self, *args, **kwargs):
"""essentially calls init of orderdict, and sets internal variables to default"""
if "unique_vertex_attribute" in kwargs:
self.unique_vertex_attribute = kwargs["unique_vertex_attribute"]
del kwargs["unique_vertex_attribute"]
else:
logger.warning("unique_vertex_attribute not set, thus it will be set to 'label'")
self.unique_vertex_attribute = "label" # identifies uniquely a node in a layers.
self._multi_layer = None ##contains the network created by joining the layers into one.
self._multi_layer_v_attr_name = None
self._ynws = {} ##contains a multilayergraph for each year
# _components = set()
self._intervall = 1 # number of years merged in one year
......@@ -47,9 +55,18 @@ class MultiLayerGraph(OrderedDict):
self._multilayer_fig = None
self._ynws_multilayer_figs = None
self._merged_ynws = None
self._notebook_mode = False
super().__init__(*args, **kwargs)
def init_notebook_mode(self):
"""
if called tqdm_notebook is used instead of tqdm
:return:
"""
self._notebook_mode = True
class BiPartiteGraph(igraph.Graph):
"""
class for describing bi-partite layers in our network.
......@@ -148,9 +165,15 @@ class MultiLayerGraph(OrderedDict):
layer_start.vs[attribute_name] = ""
for e in tqdm(self.es):
s = self.vs["label"][e.source]
t = self.vs["label"][e.target]
if self._notebook_mode:
tqdm_local = tqdm_notebook
else:
tqdm_local = tqdm
for e in tqdm_local(self.es):
s = self.vs[self._multi_layer_v_attr_name][e.source]
t = self.vs[self._multi_layer_v_attr_name][e.target]
s_ls = layer_start.vs.select(label=s)
val = t
......@@ -167,8 +190,8 @@ class MultiLayerGraph(OrderedDict):
s_l[attribute_name] = "%s||%s" % (s_l[attribute_name], val)
if layer_start == self: ## wenn der Graph selbst benutzt wird dann auch andersherum
s = self.vs["label"][e.target]
t = self.vs["label"][e.source]
s = self.vs[self._multi_layer_v_attr_name][e.target]
t = self.vs[self._multi_layer_v_attr_name][e.source]
if layer_start == None:
layer_start = self
......@@ -230,10 +253,24 @@ class MultiLayerGraph(OrderedDict):
"""
def __init__(self, *args, chooseOnlyNodesInYear=False,
mergeattr="label",
mergeattr=None,
start_attr="begins",
end_attr="ends",
**kwargs):
"""
:param args:
:param chooseOnlyNodesInYear:
:param mergeattr:
:param start_attr:
:param end_attr:
:param kwargs:
"""
if mergeattr is None:
raise ValueError("mergeattr has to be set.")
self.init_params(chooseOnlyNodesInYear, mergeattr, start_attr, end_attr)
super().__init__()
......@@ -249,7 +286,15 @@ class MultiLayerGraph(OrderedDict):
An extended dictionary which contains a MultiLayerGraphObject for each year.
"""
def __init__(self):
def __init__(self,**kwargs):
if "unique_vertex_attribute" in kwargs:
self.unique_vertex_attribute = kwargs["unique_vertex_attribute"]
del kwargs["unique_vertex_attribute"]
else:
logger.warning("unique_vertex_attribute not set, thus it will be set to 'label'")
self.unique_vertex_attribute = "label" # identifies uniquely a node in a layers.
self._start = None
self._end = None
self._startyear_attr = None
......@@ -268,7 +313,7 @@ class MultiLayerGraph(OrderedDict):
self._endyear_edge_attr = endyear_edge_attr
def __getitem__(self, item): # immitate behaviour of defaultdict
value = super().get(item, MultiLayerGraph())
value = super().get(item, MultiLayerGraph(unique_vertex_attribute = self.unique_vertex_attribute))
self[item] = value
return value
......@@ -290,9 +335,9 @@ class MultiLayerGraph(OrderedDict):
def filter_edge_by_vertices(self,start_vertex,end_vertex):
if isinstance(start_vertex,str):
start_vertex = {"label":start_vertex}
start_vertex = {self._multi_layer_v_attr_name:start_vertex}
if isinstance(end_vertex,str):
end_vertex = {"label":end_vertex}
end_vertex = {self._multi_layer_v_attr_name:end_vertex}
nw = self.copy()
for y, ynw in self.items():
......@@ -462,6 +507,13 @@ class MultiLayerGraph(OrderedDict):
"multi_layer not calculated yet. Please run generateMultiLayer(self, v_attr_name) first!")
return self._multi_layer
def getLayerTypes(self,field="typ"):
"""get the unique set of typs in each layer"""
ret ={}
for v,nw in self.items():
ret[v] = set(nw.vs[field])
return ret
def addLayerName(self):
"""
add to each graph in the system a label "layer" to the edges.
......@@ -471,7 +523,7 @@ class MultiLayerGraph(OrderedDict):
for n, gr in self.items():
gr.es["layer"] = n
def generateMultiLayer(self, v_attr_name="label", update=False):
def generateMultiLayer(self, update=False):
"""
generates one graph which combines all nodes and edges of the multlayer structure.
:param v_attr_name: (optional) defaults to label. Name of the attribute of the
......@@ -479,12 +531,14 @@ class MultiLayerGraph(OrderedDict):
:param update: (optional) experimental if set to true it only adds new layers
:return:
"""
v_attr_name = self.unique_vertex_attribute
if not update:
multi_layer = MultiLayerGraph.JoinedMultiLayerGraph()
multi_layer.vs[v_attr_name] = ""
else:
multi_layer = self._multi_layer
v_attr_name = self._multi_layer_v_attr_name
logging.debug("generate multilayer: components before: %s" % multi_layer._components)
for k, gr in self.items():
......@@ -500,7 +554,6 @@ class MultiLayerGraph(OrderedDict):
else:
logging.debug("not added (no vertices): %s" % multi_layer._components)
self._multi_layer = multi_layer
self._multi_layer_v_attr_name = v_attr_name
logging.debug("generate multilayer: components now: %s" % multi_layer._components)
return True
......@@ -541,7 +594,7 @@ class MultiLayerGraph(OrderedDict):
for p in vertices:
try:
pc = gr.vs.find(label=p["label"])
pc = gr.vs.find(**{self.unique_vertex_attribute:p[self.unique_vertex_attribute]})
pc_betw = getattr(pc, func)()
gr_betw = getattr(gr, func)()
......@@ -552,15 +605,37 @@ class MultiLayerGraph(OrderedDict):
return sizes
def createFigLayeredGraph(self, name, layers, sizes, only_in_layer=False, layer_typ={}):
def createFigLayeredGraph(self, name, layers, sizes,
position_per_layer=False,
layer_typ=None,
vertex_label_attr=None):
"""
:param name: name/title of the figure
:param layers: list of layer to be displayed, layers are displayed in the order given in the list
:param sizes: size of the vertices
:param position_per_layer: defaults to false, show only the point which exist in the layer
:param layer_typ: dictionary layer -> types to be displayed in this layer. defaults to all type in the layer.
:param vertex_label_attr:
:return:
"""
if layer_typ is None:
layer_typ = self.getLayerTypes()
if vertex_label_attr is None:
vertex_label_attr = self.unique_vertex_attribute
layer_graphs = self
pos_ml = self.multi_layer.layout_fruchterman_reingold()
poss = {}
if only_in_layer:
if position_per_layer:
for l, gr in self.items():
poss[l] = gr.layout_fruchterman_reingold()
else:
pos_ml = self.multi_layer.layout_fruchterman_reingold()
# generate links between the layers
......@@ -568,25 +643,25 @@ class MultiLayerGraph(OrderedDict):
Yl = []
Zl = []
for n in self.multi_layer.vs:
label = n["label"]
label = n[self.unique_vertex_attribute]
for layer_num in range(0, len(layers) - 1):
layer = layers[layer_num]
layer_gr = layer_graphs[layer]
if layer_typ: # only relevant if only_in_layer is true
if layer_typ: # only relevant if position_per_layer is true
if not n["typ"] in layer_typ[layer]:
continue
if only_in_layer:
if position_per_layer:
pos = poss[layer]
else:
pos = pos_ml
if only_in_layer:
if position_per_layer:
try:
n_tmp = layer_gr.vs.find(label=label) # TODO search also for typ
n_tmp = layer_gr.vs.find(**{self.unique_vertex_attribute:label}) # TODO search also for typ
ni = n_tmp.index
except:
continue
......@@ -600,15 +675,15 @@ class MultiLayerGraph(OrderedDict):
layer = layers[layer_num + 1]
layer_gr = layer_graphs[layer]
if only_in_layer:
if position_per_layer:
pos = poss[layer]
else:
pos = pos_ml
if only_in_layer:
if position_per_layer:
try:
n_tmp = layer_gr.vs.find(label=label) # TODO search also for typ
n_tmp = layer_gr.vs.find(**{self.unique_vertex_attribute:label}) # TODO search also for typ
ni = n_tmp.index
except:
continue
......@@ -646,7 +721,7 @@ class MultiLayerGraph(OrderedDict):
logger.warning("Layer %s not in the graph, cannot display it." % layer)
continue
if only_in_layer:
if position_per_layer:
pos = poss[layer]
......@@ -665,12 +740,12 @@ class MultiLayerGraph(OrderedDict):
if e.source == e.target:
continue
if only_in_layer:
if position_per_layer:
es = e.source
et = e.target
else:
es = self.multi_layer.vs.find(label=layer_gr.vs[e.source]["label"]).index
et = self.multi_layer.vs.find(label=layer_gr.vs[e.target]["label"]).index
es = self.multi_layer.vs.find(**{self.unique_vertex_attribute:layer_gr.vs[e.source][self.unique_vertex_attribute]}).index
et = self.multi_layer.vs.find(**{self.unique_vertex_attribute:layer_gr.vs[e.target][self.unique_vertex_attribute]}).index
Xe += [pos[es][0], pos[et][0], None] # x-coordinates of edge ends
Ye += [pos[es][1], pos[et][1], None]
......@@ -704,7 +779,7 @@ class MultiLayerGraph(OrderedDict):
logger.warning("Layer %s not in the graph, cannot display it." % layer)
continue
if only_in_layer:
if position_per_layer:
pos = poss[layer]
gr = self[layer]
else:
......@@ -715,7 +790,7 @@ class MultiLayerGraph(OrderedDict):
Yn += [k[1] for k in pos.coords] # x-coordinates of nodes
# Zn += [layers.index(layer) for k in self.multi_layer.vs]
Zn += [layer for k in gr.vs]
text += gr.vs["label"]
text += gr.vs[vertex_label_attr]
color += [scale[layers.index(layer)] for x in pos.coords]
if isinstance(sizes, dict):
......@@ -800,8 +875,8 @@ class MultiLayerGraph(OrderedDict):
try:
for v in self.multi_layer.vs:
self._all_nodes[(v["label"], v["typ"])].update(v.attributes())
del self._all_nodes[(v["label"], v["typ"])]["id"] # id is specific for each graph and shouldn't be here
self._all_nodes[(v[self.unique_vertex_attribute], v["typ"])].update(v.attributes())
del self._all_nodes[(v[self.unique_vertex_attribute], v["typ"])]["id"] # id is specific for each graph and shouldn't be here
except NotCalculatedYetError:
pass
......@@ -811,12 +886,12 @@ class MultiLayerGraph(OrderedDict):
if k == "id" or k in allow_overwrite:
continue ## id is normally graph specific
if v and not v == "None":
oldValue = self._all_nodes[(n["label"], n["typ"])].get(k, None)
oldValue = self._all_nodes[(n[self.unique_vertex_attribute], n["typ"])].get(k, None)
if oldValue == "None":
oldValue = None
if isinstance(oldValue,float) and math.isnan(oldValue):
self._all_nodes[(n["label"], n["typ"])][k] = v
self._all_nodes[(n[self.unique_vertex_attribute], n["typ"])][k] = v
elif oldValue and not oldValue == v:
try:
if float(oldValue) != float(v): # could just be the same value but different types - in the case of floats or ints
......@@ -824,19 +899,19 @@ class MultiLayerGraph(OrderedDict):
except:
raise ValueError("layer %s: not all values %s the same for: %s was (%s)" % (ln,k, n, oldValue))
else:
self._all_nodes[(n["label"], n["typ"])][k] = v
self._all_nodes[(n[self.unique_vertex_attribute], n["typ"])][k] = v
# now add this values to all graphs
try:
for v in self.multi_layer.vs:
for k, val in self._all_nodes[(v["label"], v["typ"])].items():
for k, val in self._all_nodes[(v[self.unique_vertex_attribute], v["typ"])].items():
v[k] = val
except NotCalculatedYetError:
pass
for gr in self.values():
for v in gr.vs:
for k, val in self._all_nodes[(v["label"], v["typ"])].items():
for k, val in self._all_nodes[(v[self.unique_vertex_attribute], v["typ"])].items():
v[k] = val
@property
......@@ -848,7 +923,7 @@ class MultiLayerGraph(OrderedDict):
return self._ynws
def createYearNetworks(self, start, end,
v_attr_name="label",
v_attr_name=None,
startyear_attr="begins",
endyear_attr="ends",
startyear_edge_attr="year_s",
......@@ -859,8 +934,27 @@ class MultiLayerGraph(OrderedDict):
combine_edges=None,
update=False,
worker = 6):
"""
:param start: first year
:param end: last year
:param v_attr_name: Vertex attribute which uniquely identifies the vertices in each year, if None, then self.unique_vertex_attribute is used.
:param startyear_attr: default "begins", vertex attribute which contains the year, where the vertex starts to exist.
:param endyear_attr: default "ends", vertex attribute which contains the year, where the vertex ends to exist.
:param startyear_edge_attr: default "year_s", edge attribute which contains the year, where the edge starts to exist.
:param endyear_edge_attr: default "year_e", edge attribute which contains the year, where the edge ends to exist.
:param filter_degree_0: default False, filter_degree_0 can be either a boolean if true then filter in all graphs or a list then filter only for the graphs in the list.
:param include_nodes_without_date: default False, vertices without dates are not displayed if false.
:param simplify: calls simplify(multiple=True, loops=True, combine_edges=combine_edges) on each network
:param combine_edges: see above, defaults to None
:param update: defaults False, if true create only years/networks which don't exist alredy
:param worker: defaults to 6, number of workers to use for parallel processing of creating the networks.
:return:
"""
ynws = {}
if v_attr_name is None:
v_attr_name =self.unique_vertex_attribute
for v, gr in self.items():
# if update:
......@@ -881,20 +975,32 @@ class MultiLayerGraph(OrderedDict):
logging.warning("%s not in edge attributes of graph %s !" % (startyear_edge_attr, v))
logger.debug("%s type %s" % (v, type(gr)))
if self._notebook_mode:
tqdm_local = tqdm_notebook
else:
tqdm_local = tqdm
#filter_degreee_0 can eitehr be a boolean if true then filter everywhere
#or a list then filter only for the graphs in the list.
if not isinstance(filter_degree_0,bool):
filter_degree_0_local = v in filter_degree_0
else: #
filter_degree_0_local = filter_degree_0
ynws[v] = igraphx.createYearNetworksParallel(gr, start, end,
tqdm=tqdm,
tqdm=tqdm_local,
startyear_attr=startyear_attr,
endyear_attr=endyear_attr,
startyear_edge_attr=startyear_edge_attr,
endyear_edge_attr=endyear_edge_attr,
filter_degree_0=filter_degree_0,
filter_degree_0=filter_degree_0_local,
include_nodes_without_date=include_nodes_without_date,
simplify=simplify,
combine_edges=combine_edges,
worker = worker)
if not update:
self._ynws = MultiLayerGraph.YearNetwork()
self._ynws = MultiLayerGraph.YearNetwork(unique_vertex_attribute = self.unique_vertex_attribute)
self._ynws.init_params(start,
end,
......@@ -906,9 +1012,9 @@ class MultiLayerGraph(OrderedDict):
for v, ynw in ynws.items():
for y, gr in ynw.items():
nw = self._ynws.get(y, MultiLayerGraph())
nw = self._ynws.get(y, MultiLayerGraph(unique_vertex_attribute = self.unique_vertex_attribute))
nw[v] = gr
nw.generateMultiLayer(v_attr_name=v_attr_name)
nw.generateMultiLayer()
self._ynws[y] = nw
return True
......@@ -919,9 +1025,11 @@ class MultiLayerGraph(OrderedDict):
size_offset=5,
bipartite=False,
years_selected=None,
only_in_layer=False,
position_per_layer=False,
disable=True,
merged_years = None):
merged_years = None,
layer_typ = None,
vertex_label_attr=None):
"""
:param name:
......@@ -931,7 +1039,7 @@ class MultiLayerGraph(OrderedDict):
:param size_offset:
:param bipartite:
:param years_selected:
:param only_in_layer:
:param position_per_layer:
:param disable:
:param merged_years:
:return:
......@@ -950,17 +1058,39 @@ class MultiLayerGraph(OrderedDict):
else:
ynw = self.merged_ynws[merged_years].ynws
for y, nw in tqdm(ynw.items(), disable=disable):
if self._notebook_mode:
tqdm_local = tqdm_notebook
else:
tqdm_local = tqdm
for y, nw in tqdm_local(ynw.items(), disable=disable):
if years_selected and not y in years_selected:
logger.info("skip: %s"%y)
continue
sizes = nw.calc_sizes(size_mult=size_mult, size_offset=size_offset, func=func)
if bipartite:
logger.debug(("bipartite mode"))
nw.createFigLayeredGraphBip(name, layers, sizes, only_in_layer=only_in_layer)
try:
nw.createFigLayeredGraphBip(name, layers, sizes,
position_per_layer=position_per_layer,
layer_typ=layer_typ,
vertex_label_attr=vertex_label_attr)
except NotCalculatedYetError:
logging.info(" multi_layer not calculated yet for year %s. Will not be displayed." %y)
continue
else:
logger.debug(("non bipartite mode"))
nw.createFigLayeredGraph(name, layers, sizes, only_in_layer=only_in_layer)
try:
nw.createFigLayeredGraph(name, layers, sizes,
position_per_layer=position_per_layer,
layer_typ=layer_typ,
vertex_label_attr=vertex_label_attr)
except NotCalculatedYetError:
logging.info(" multi_layer not calculated yet for year %s. Will not be displayed." %y)
continue
figs[y] = nw.multilayer_fig
self._ynws_multilayer_figs = figs